Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.28169543 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.023758385 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.023467196 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1192 1193 1194 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.473363623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.54809082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.549495882Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.549624885Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.549651197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.549669699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196} {"msg":"Crucible 485d2f03-a356-4d5e-83b3-a5530957e750 has session id: e4d77780-2b22-46a6-b4ec-6e4c532be368","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.550547488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Upstairs opts: Upstairs UUID: 485d2f03-a356-4d5e-83b3-a5530957e750, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.550685832Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Crucible stats registered with UUID: 485d2f03-a356-4d5e-83b3-a5530957e750","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.550721756Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:03.550788702Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552023427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552122327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196} {"msg":"485d2f03-a356-4d5e-83b3-a5530957e750 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552143289Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.55215696Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"0","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552205255Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"0","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552223457Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"1","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.55224817Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"1","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552281613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"2","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552308656Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"2","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552331078Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.55235342Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552376992Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"ds_connection connected from Ok(127.0.0.1:33279)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552537259Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"io task","client":"1","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"ds_connection connected from Ok(127.0.0.1:59134)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552571132Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"io task","client":"2","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"ds_connection connected from Ok(127.0.0.1:41490)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.552597055Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"io task","client":"0","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.554260113Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"0","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.554385175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"1","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.554426469Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"2","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.555412799Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"1","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.555451813Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"2","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.555477975Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"0","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556628481Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556662875Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556839302Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556859475Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556874156Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556888138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556902249Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.55691616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556929602Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556942743Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556961145Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556974446Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.556987468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557023211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557036893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"":"downstairs","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557049774Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557061535Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"0","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557074726Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"1","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557087548Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"client":"2","":"downstairs","session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557100299Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.55711221Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"485d2f03-a356-4d5e-83b3-a5530957e750 is now active with session: e4d77780-2b22-46a6-b4ec-6e4c532be368","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557124951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557136623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196,"session_id":"e4d77780-2b22-46a6-b4ec-6e4c532be368"} {"msg":Wait for a query_work_queue command to finish before sending IO "The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:03.557154054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:07.647846269Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1196} Begin loop 1 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.679758808Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.725872619Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.726624045Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.726667479Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.7266831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.726698022Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198} {"msg":"Crucible 7a96bc73-3a08-4ed0-a661-5595a30145cd has session id: 436e6c7c-2466-4055-b68a-656652af5009","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.727231936Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Upstairs opts: Upstairs UUID: 7a96bc73-3a08-4ed0-a661-5595a30145cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.72727142Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Crucible stats registered with UUID: 7a96bc73-3a08-4ed0-a661-5595a30145cd","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.727293632Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:07.727312784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.727793472Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.727824526Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198} {"msg":"7a96bc73-3a08-4ed0-a661-5595a30145cd active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.727876571Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.727908044Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"0","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.727946688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"0","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.727972831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"1","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728000664Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"1","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728024446Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"2","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728051578Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"2","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728077441Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728103384Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728129016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"ds_connection connected from Ok(127.0.0.1:64650)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728152139Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"ds_connection connected from Ok(127.0.0.1:54189)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728174891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"ds_connection connected from Ok(127.0.0.1:57502)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728200514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728837998Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"0","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.728864901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"2","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.729447289Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"2","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.729585153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"0","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.730108386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.730214367Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.730816328Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"1","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.731387825Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"1","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.731953652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.731972514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.731987155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732000737Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732014958Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.73202846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732045331Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732058633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732072164Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732084635Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732097707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732110128Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.73213487Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"":"downstairs","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732158713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732177975Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"0","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732198777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"1","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732223049Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"client":"2","":"downstairs","session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732245992Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732268844Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"7a96bc73-3a08-4ed0-a661-5595a30145cd is now active with session: 436e6c7c-2466-4055-b68a-656652af5009","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732291806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.732315549Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198,"session_id":"436e6c7c-2466-4055-b68a-656652af5009"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:07.7323291Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 20480 data: 2 2 2 2 2 02/30 Write block 469 len 20480 data: 2 2 2 2 2 03/30 Write block 165 len 4096 data: 2 04/30 Read block 263 len 32768 05/30 Read block 374 len 28672 06/30 Read block 347 len 16384 07/30 Read block 32 len 24576 08/30 Write block 412 len 4096 data: 2 09/30 Read block 411 len 40960 10/30 Write block 92 len 4096 data: 2 11/30 Write block 582 len 4096 data: 2 12/30 Write block 408 len 12288 data: 2 2 2 13/30 Flush 14/30 Write block 52 len 28672 data: 2 2 2 2 2 2 2 15/30 Read block 576 len 32768 16/30 Flush 17/30 Write block 236 len 16384 data: 2 2 2 2 18/30 Read block 129 len 12288 19/30 Flush 20/30 Write block 78 len 16384 data: 3 3 3 2 21/30 Read block 361 len 36864 22/30 Read block 448 len 36864 23/30 Read block 543 len 40960 24/30 Read block 479 len 4096 25/30 Flush 26/30 Write block 0 len 4096 data: 2 27/30 Write block 497 len 20480 data: 2 2 2 2 2 28/30 Write block 348 len 12288 data: 2 2 2 29/30 Read block 396 len 24576 30/30 Write block 309 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 6 Sent Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 1 6 Done 3 2 4 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:07.750708604Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 2 2 2 5 5 5 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 4 4 4 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 T T T 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 T T T 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 T T T 25 500-519 1 1 1 1 1 1 T T T 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 2, Max flush: 5 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 2 2 2 5 5 5 F F F 4 080-099 2 2 2 5 5 5 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 4 4 4 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 3 3 3 6 6 6 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 3 3 3 6 6 6 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 3 3 3 6 6 6 F F F 25 500-519 3 3 3 6 6 6 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.010571349Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.055268187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.056220143Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.056271118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.05628851Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.056306282Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206} {"msg":"Crucible 2ab9f1b6-0a69-4c48-91e2-b49305d12d91 has session id: 30723c57-63ad-44a5-a5b2-ca6208238922","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.05688198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Upstairs opts: Upstairs UUID: 2ab9f1b6-0a69-4c48-91e2-b49305d12d91, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.056923044Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Crucible stats registered with UUID: 2ab9f1b6-0a69-4c48-91e2-b49305d12d91","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.056951016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:12.056972659Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057310543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057334305Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206} {"msg":"2ab9f1b6-0a69-4c48-91e2-b49305d12d91 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057396961Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057414173Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"0","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057436465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"0","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057450007Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"1","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057463728Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"1","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.05747853Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057504062Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057529105Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057555017Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.0575804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"ds_connection connected from Ok(127.0.0.1:35314)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057714064Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:12.057865349Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:12.057906153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"ds_connection connected from Ok(127.0.0.1:64413)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057928615Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:12.057953108Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.057967499Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.058440087Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"1","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.05847704Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"0","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.059085472Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"0","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.059121395Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"1","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.059733287Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:12.05975937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.059052518Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"ds_connection connected from Ok(127.0.0.1:34193)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.059202153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.060311075Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061032868Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[0]R flush_numbers[0..12]: [6, 1, 3, 5, 5, 1, 1, 1, 2, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061719177Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[0]R generation[0..12]: [3, 1, 2, 2, 2, 1, 1, 1, 2, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061738689Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061753621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[1]R flush_numbers[0..12]: [6, 1, 3, 5, 5, 1, 1, 1, 2, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061766932Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[1]R generation[0..12]: [3, 1, 2, 2, 2, 1, 1, 1, 2, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061780584Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061794565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[2]R flush_numbers[0..12]: [6, 1, 3, 5, 5, 1, 1, 1, 2, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061808246Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[2]R generation[0..12]: [3, 1, 2, 2, 2, 1, 1, 1, 2, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061821308Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061836139Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.06184984Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061866822Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061882654Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061898425Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"":"downstairs","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061924998Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061939569Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"0","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061955841Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"1","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061971683Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"client":"2","":"downstairs","session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.061987694Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.062002436Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"2ab9f1b6-0a69-4c48-91e2-b49305d12d91 is now active with session: 30723c57-63ad-44a5-a5b2-ca6208238922","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.062018397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.062032849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206,"session_id":"30723c57-63ad-44a5-a5b2-ca6208238922"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:22.06204758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 86 len 36864 data: 2 2 2 2 2 2 3 2 2 03/30 Write block 139 len 20480 data: 2 2 2 2 2 04/30 Read block 515 len 20480 05/30 Write block 523 len 36864 data: 2 2 2 2 2 2 2 2 2 06/30 Write block 332 len 36864 data: 2 2 2 2 2 2 2 2 2 07/30 Write block 413 len 8192 data: 2 2 08/30 Read block 486 len 16384 09/30 Write block 499 len 4096 data: 3 10/30 Read block 270 len 32768 11/30 Write block 318 len 24576 data: 2 2 2 2 2 2 12/30 Read block 5 len 40960 13/30 Read block 282 len 8192 14/30 Flush 15/30 Write block 365 len 4096 data: 2 16/30 Read block 50 len 24576 17/30 Write block 75 len 28672 data: 2 3 3 4 4 4 3 18/30 Read block 500 len 4096 19/30 Write block 586 len 8192 data: 2 2 20/30 Read block 583 len 40960 21/30 Write block 184 len 12288 data: 2 2 2 22/30 Write block 412 len 8192 data: 3 3 23/30 Read block 204 len 16384 24/30 Flush 25/30 Write block 467 len 20480 data: 2 2 3 3 3 26/30 Read block 24 len 16384 27/30 Read block 594 len 20480 28/30 Read block 412 len 20480 29/30 Write block 371 len 36864 data: 2 2 2 2 2 2 2 2 2 30/30 Read block 66 len 24576 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 6 5 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:22.082606794Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 4 4 4 8 8 8 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 4 4 4 9 9 9 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 4 4 4 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 4 4 4 8 8 8 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 4 4 4 9 9 9 T T T 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 2 2 2 T T T 24 480-499 4 4 4 8 8 8 F F F 25 500-519 3 3 3 6 6 6 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 9 9 9 F F F Max gen: 4, Max flush: 9 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 6 6 6 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 4 4 4 8 8 8 F F F 7 140-159 4 4 4 8 8 8 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 4 4 4 9 9 9 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 4 4 4 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 4 4 4 8 8 8 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 5 5 5 10 10 10 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 3 3 3 6 6 6 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 9 9 9 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.306858919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.355187664Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.356304206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.356350861Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.356365763Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.356384954Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214} {"msg":"Crucible 6ac5ab1a-8f9d-40cf-8b08-b3735bd50bbe has session id: a49110e3-13df-4ed0-aec5-0ebbd4000d26","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.35703738Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Upstairs opts: Upstairs UUID: 6ac5ab1a-8f9d-40cf-8b08-b3735bd50bbe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357086535Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Crucible stats registered with UUID: 6ac5ab1a-8f9d-40cf-8b08-b3735bd50bbe","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357114248Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:26.357139871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.35763053Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357654062Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214} {"msg":"6ac5ab1a-8f9d-40cf-8b08-b3735bd50bbe active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357721239Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357738931Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"0","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357766944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"0","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357780665Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"1","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357794867Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"1","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357808018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"2","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.35782185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"2","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357839981Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.357865564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.358052473Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"ds_connection connected from Ok(127.0.0.1:36339)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.358088316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"ds_connection connected from Ok(127.0.0.1:41924)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.358112809Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"ds_connection connected from Ok(127.0.0.1:48144)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.358134531Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.358732631Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"1","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.358764385Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"0","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.359502419Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"1","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.359554124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"0","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.36000462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.360223812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.360367786Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"2","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.360879908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"2","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[0]R flush_numbers[0..12]: [6, 1, 3, 9, 9, 1, 8, 8, 2, 9, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361428263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[0]R generation[0..12]: [3, 1, 2, 4, 4, 1, 4, 4, 2, 4, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361448805Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361464047Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[1]R flush_numbers[0..12]: [6, 1, 3, 9, 9, 1, 8, 8, 2, 9, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361477648Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[1]R generation[0..12]: [3, 1, 2, 4, 4, 1, 4, 4, 2, 4, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.36149074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361504591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[2]R flush_numbers[0..12]: [6, 1, 3, 9, 9, 1, 8, 8, 2, 9, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361518092Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[2]R generation[0..12]: [3, 1, 2, 4, 4, 1, 4, 4, 2, 4, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361531184Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361544555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361560596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361580198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.36159279Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361605271Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"":"downstairs","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361617932Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361629523Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"0","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361642195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"1","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361654916Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"client":"2","":"downstairs","session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361667707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361679488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"6ac5ab1a-8f9d-40cf-8b08-b3735bd50bbe is now active with session: a49110e3-13df-4ed0-aec5-0ebbd4000d26","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.3616918Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361705131Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214,"session_id":"a49110e3-13df-4ed0-aec5-0ebbd4000d26"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:26.361717282Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 383 len 28672 data: 2 2 2 2 2 2 2 02/30 Write block 71 len 24576 data: 2 2 2 2 3 4 03/30 Read block 99 len 24576 04/30 Write block 461 len 40960 data: 2 2 2 2 2 2 3 3 4 4 05/30 Read block 410 len 20480 06/30 Read block 260 len 36864 07/30 Write block 311 len 12288 data: 2 2 2 08/30 Read block 467 len 8192 09/30 Flush 10/30 Write block 27 len 12288 data: 2 2 2 11/30 Flush 12/30 Write block 201 len 8192 data: 2 2 13/30 Write block 215 len 36864 data: 2 2 2 2 2 2 2 2 2 14/30 Write block 509 len 20480 data: 2 2 2 2 2 15/30 Write block 15 len 8192 data: 2 2 16/30 Flush 17/30 Write block 17 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Read block 155 len 32768 19/30 Read block 358 len 4096 20/30 Write block 26 len 16384 data: 2 3 3 3 21/30 Read block 392 len 24576 22/30 Read block 224 len 40960 23/30 Read block 391 len 24576 24/30 Write block 117 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Write block 199 len 32768 data: 2 2 3 3 2 2 2 2 26/30 Write block 63 len 12288 data: 2 2 2 27/30 Read block 489 len 28672 28/30 Write block 149 len 4096 data: 2 29/30 Write block 294 len 16384 data: 2 2 2 2 30/30 Write block 300 len 20480 data: 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 9 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:26.380870544Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 13 13 13 T T T 1 020-039 6 6 6 12 12 12 T T T 2 040-059 2 2 2 3 3 3 F F F 3 060-079 6 6 6 11 11 11 T T T 4 080-099 4 4 4 9 9 9 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 4 4 4 8 8 8 T T T 7 140-159 4 4 4 8 8 8 T T T 8 160-179 2 2 2 2 2 2 F F F 9 180-199 4 4 4 9 9 9 T T T 10 200-219 6 6 6 13 13 13 T T T 11 220-239 6 6 6 13 13 13 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 T T T 15 300-319 6 6 6 11 11 11 T T T 16 320-339 4 4 4 8 8 8 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 6 6 6 11 11 11 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 6 6 6 13 13 13 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 9 9 9 F F F Max gen: 6, Max flush: 13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 14 14 14 F F F 1 020-039 7 7 7 14 14 14 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 7 7 7 14 14 14 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 7 7 7 14 14 14 F F F 6 120-139 7 7 7 14 14 14 F F F 7 140-159 7 7 7 14 14 14 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 6 6 6 13 13 13 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 7 7 7 14 14 14 F F F 15 300-319 7 7 7 14 14 14 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 6 6 6 11 11 11 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 6 6 6 13 13 13 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 9 9 9 F F F Max gen: 7, Max flush: 14 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.649020497Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:40.693448548Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.693946608Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.726444835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.727233995Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.727268089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.72728124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.727295231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222} {"msg":"Crucible c2154144-c08e-44b2-b750-ada6027f9a3e has session id: 95f67f9f-fcf8-44b0-95e0-6b7538100e80","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.727894212Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Upstairs opts: Upstairs UUID: c2154144-c08e-44b2-b750-ada6027f9a3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.727927405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Crucible stats registered with UUID: c2154144-c08e-44b2-b750-ada6027f9a3e","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.727943687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:40.727956548Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728350978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.72837578Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222} {"msg":"c2154144-c08e-44b2-b750-ada6027f9a3e active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728442007Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728459229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"0","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728482461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"0","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728496953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"1","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728511224Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"1","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728524685Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"2","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728538827Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"2","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728551598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.7285652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728592462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"ds_connection connected from Ok(127.0.0.1:51460)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728623185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"ds_connection connected from Ok(127.0.0.1:58884)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.728974891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"ds_connection connected from Ok(127.0.0.1:47930)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.729045298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.729355869Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"2","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.729382762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"1","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.729636777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"0","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730112045Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"1","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730168621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"2","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730188503Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"0","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730664031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730783753Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[0]R flush_numbers[0..12]: [14, 14, 3, 14, 9, 14, 14, 14, 2, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730866881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[0]R generation[0..12]: [7, 7, 2, 7, 4, 7, 7, 7, 2, 7, 7, 6]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730885194Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730902525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[1]R flush_numbers[0..12]: [14, 14, 3, 14, 9, 14, 14, 14, 2, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730916337Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[1]R generation[0..12]: [7, 7, 2, 7, 4, 7, 7, 7, 2, 7, 7, 6]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730931618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.73094905Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[2]R flush_numbers[0..12]: [14, 14, 3, 14, 9, 14, 14, 14, 2, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730965831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[2]R generation[0..12]: [7, 7, 2, 7, 4, 7, 7, 7, 2, 7, 7, 6]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.730983093Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731000005Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731015717Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731037169Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.7310529Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731068562Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"":"downstairs","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731084323Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731098785Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"0","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731114626Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"1","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731130428Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"client":"2","":"downstairs","session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:23:40.7311489Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731167102Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"c2154144-c08e-44b2-b750-ada6027f9a3e is now active with session: 95f67f9f-fcf8-44b0-95e0-6b7538100e80","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731182993Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731197565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222,"session_id":"95f67f9f-fcf8-44b0-95e0-6b7538100e80"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:40.731212296Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 349 len 4096 data: 3 03/30 Read block 151 len 16384 04/30 Write block 67 len 20480 data: 2 2 2 2 3 05/30 Read block 246 len 28672 06/30 Flush 07/30 Write block 86 len 24576 data: 3 3 3 3 3 3 08/30 Write block 425 len 16384 data: 2 2 2 2 09/30 Write block 243 len 32768 data: 2 2 2 2 2 2 2 2 10/30 Read block 184 len 12288 11/30 Read block 96 len 12288 12/30 Write block 506 len 4096 data: 2 13/30 Flush 14/30 Write block 117 len 24576 data: 3 3 3 3 3 3 15/30 Write block 541 len 36864 data: 2 2 2 2 2 2 2 2 2 16/30 Read block 327 len 12288 17/30 Read block 148 len 32768 18/30 Read block 342 len 12288 19/30 Write block 55 len 16384 data: 3 3 3 3 20/30 Flush 21/30 Write block 583 len 12288 data: 2 2 2 22/30 Read block 474 len 36864 23/30 Write block 529 len 12288 data: 3 3 3 24/30 Write block 585 len 28672 data: 3 3 3 2 2 2 2 25/30 Read block 155 len 32768 26/30 Write block 421 len 32768 data: 2 2 2 2 3 3 3 3 27/30 Read block 56 len 40960 28/30 Read block 278 len 12288 29/30 Write block 225 len 28672 data: 2 2 2 2 2 2 2 30/30 Write block 425 len 24576 data: 4 4 4 4 2 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:40.750373899Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 14 14 14 F F F 1 020-039 7 7 7 14 14 14 F F F 2 040-059 8 8 8 18 18 18 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 8 8 8 17 17 17 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 7 7 7 14 14 14 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 6 6 6 13 13 13 T T T 12 240-259 8 8 8 17 17 17 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 7 7 7 14 14 14 F F F 15 300-319 7 7 7 14 14 14 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 8 8 8 16 16 16 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 8 8 8 17 17 17 T T T 22 440-459 1 1 1 1 1 1 F F F 23 460-479 6 6 6 11 11 11 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 F F F 26 520-539 4 4 4 8 8 8 T T T 27 540-559 8 8 8 18 18 18 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 9 9 9 T T T Max gen: 8, Max flush: 18 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 14 14 14 F F F 1 020-039 7 7 7 14 14 14 F F F 2 040-059 8 8 8 18 18 18 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 8 8 8 17 17 17 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 8 8 8 18 18 18 F F F 7 140-159 7 7 7 14 14 14 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 8 8 8 17 17 17 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 7 7 7 14 14 14 F F F 15 300-319 7 7 7 14 14 14 F F F 16 320-339 4 4 4 8 8 8 F F F 17 340-359 8 8 8 16 16 16 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 4 4 4 9 9 9 F F F 21 420-439 9 9 9 19 19 19 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 6 6 6 11 11 11 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 F F F 26 520-539 9 9 9 19 19 19 F F F 27 540-559 8 8 8 18 18 18 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 9 9 9 19 19 19 F F F Max gen: 9, Max flush: 19 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.016056382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.060836718Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.061636249Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.061683404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.061698465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.061727728Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230} {"msg":"Crucible c2e61847-47bc-4afa-9fea-310afb31d24a has session id: 07edc182-cbd5-4d8c-a303-ea989ae2f528","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.062308077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Upstairs opts: Upstairs UUID: c2e61847-47bc-4afa-9fea-310afb31d24a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.062349121Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Crucible stats registered with UUID: c2e61847-47bc-4afa-9fea-310afb31d24a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.062374303Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:45.062395525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.062894466Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.062930519Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230} {"msg":"c2e61847-47bc-4afa-9fea-310afb31d24a active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.062997536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063022069Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"0","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063047011Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"0","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063062903Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"1","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063077474Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"1","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063094126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063117588Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063134Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063155492Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063177704Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"ds_connection connected from Ok(127.0.0.1:35736)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063566514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"ds_connection connected from Ok(127.0.0.1:47039)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063610878Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:45.063645002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:45.063675875Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:45.063706238Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.063723239Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.064210448Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"0","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.064240562Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"1","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.064886197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"0","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.06491611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"1","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.065560835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:45.065593008Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.06433919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"ds_connection connected from Ok(127.0.0.1:38644)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.064480375Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.06572565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.066396348Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[0]R flush_numbers[0..12]: [14, 14, 18, 16, 17, 18, 18, 14, 2, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067031402Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[0]R generation[0..12]: [7, 7, 8, 8, 8, 8, 8, 7, 2, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067055644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067072166Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[1]R flush_numbers[0..12]: [14, 14, 18, 16, 17, 18, 18, 14, 2, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067087118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[1]R generation[0..12]: [7, 7, 8, 8, 8, 8, 8, 7, 2, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067101029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.06711506Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[2]R flush_numbers[0..12]: [14, 14, 18, 16, 17, 18, 18, 14, 2, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067128612Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[2]R generation[0..12]: [7, 7, 8, 8, 8, 8, 8, 7, 2, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067146064Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067160415Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067178417Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067191468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067204049Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.06721636Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"":"downstairs","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067229102Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067240993Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"0","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067253944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"1","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067266796Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"client":"2","":"downstairs","session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067279687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067291658Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"c2e61847-47bc-4afa-9fea-310afb31d24a is now active with session: 07edc182-cbd5-4d8c-a303-ea989ae2f528","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067304169Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.06731581Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230,"session_id":"07edc182-cbd5-4d8c-a303-ea989ae2f528"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:55.067327962Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 112 len 36864 data: 2 2 2 2 2 4 4 4 4 02/30 Read block 72 len 16384 03/30 Read block 498 len 16384 04/30 Write block 439 len 4096 data: 2 05/30 Read block 513 len 8192 06/30 Flush 07/30 Write block 105 len 32768 data: 2 2 2 2 2 2 2 3 08/30 Flush 09/30 Write block 110 len 40960 data: 3 3 4 3 3 3 3 5 5 5 10/30 Read block 240 len 20480 11/30 Write block 45 len 28672 data: 2 2 2 2 2 2 2 12/30 Read block 556 len 32768 13/30 Read block 245 len 24576 14/30 Write block 559 len 4096 data: 2 15/30 Flush 16/30 Write block 117 len 40960 data: 6 6 6 5 4 4 3 3 2 2 17/30 Read block 500 len 36864 18/30 Read block 72 len 4096 19/30 Write block 151 len 16384 data: 2 2 2 2 20/30 Write block 250 len 24576 data: 3 2 2 2 2 2 21/30 Write block 405 len 24576 data: 2 2 2 3 3 3 22/30 Read block 312 len 4096 23/30 Read block 290 len 20480 24/30 Flush 25/30 Write block 286 len 28672 data: 2 2 2 2 2 2 2 26/30 Read block 458 len 16384 27/30 Write block 338 len 28672 data: 3 3 3 2 2 2 2 28/30 Write block 329 len 32768 data: 2 2 2 3 3 3 3 3 29/30 Write block 263 len 32768 data: 2 2 2 2 2 2 2 2 30/30 Read block 572 len 20480 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 5 6 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:55.088152572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 14 14 14 F F F 1 020-039 7 7 7 14 14 14 F F F 2 040-059 10 10 10 22 22 22 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 8 8 8 17 17 17 F F F 5 100-119 10 10 10 23 23 23 F F F 6 120-139 10 10 10 23 23 23 F F F 7 140-159 10 10 10 23 23 23 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 10 10 10 23 23 23 F F F 13 260-279 1 1 1 1 1 1 T T T 14 280-299 7 7 7 14 14 14 T T T 15 300-319 7 7 7 14 14 14 F F F 16 320-339 4 4 4 8 8 8 T T T 17 340-359 8 8 8 16 16 16 T T T 18 360-379 5 5 5 10 10 10 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 10 10 10 23 23 23 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 6 6 6 11 11 11 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 F F F 26 520-539 9 9 9 19 19 19 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 9 9 9 19 19 19 F F F Max gen: 10, Max flush: 23 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 14 14 14 F F F 1 020-039 7 7 7 14 14 14 F F F 2 040-059 10 10 10 22 22 22 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 8 8 8 17 17 17 F F F 5 100-119 10 10 10 23 23 23 F F F 6 120-139 10 10 10 23 23 23 F F F 7 140-159 10 10 10 23 23 23 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 10 10 10 23 23 23 F F F 13 260-279 11 11 11 24 24 24 F F F 14 280-299 11 11 11 24 24 24 F F F 15 300-319 7 7 7 14 14 14 F F F 16 320-339 11 11 11 24 24 24 F F F 17 340-359 11 11 11 24 24 24 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 6 6 6 11 11 11 F F F 20 400-419 10 10 10 23 23 23 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 6 6 6 11 11 11 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 F F F 26 520-539 9 9 9 19 19 19 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 9 9 9 19 19 19 F F F Max gen: 11, Max flush: 24 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.320864001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.365516214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.366298743Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.366356679Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.366374351Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.366389522Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238} {"msg":"Crucible 0cc4acdd-bcac-43ca-b224-703d89c18117 has session id: a4e7cc82-b9c5-4789-87a7-5481296ef1b2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.366919336Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Upstairs opts: Upstairs UUID: 0cc4acdd-bcac-43ca-b224-703d89c18117, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.366986803Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Crucible stats registered with UUID: 0cc4acdd-bcac-43ca-b224-703d89c18117","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367023196Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:59.367040148Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367509165Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367534408Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238} {"msg":"0cc4acdd-bcac-43ca-b224-703d89c18117 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367605425Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367622757Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"0","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367645399Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"0","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367669271Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367695614Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367720897Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"2","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367746629Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"2","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367768192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367790314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367812426Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63200)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367837539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63291)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.367860951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:59.368103935Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:59.368142879Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:23:59.368169352Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.368287854Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.368629018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"0","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.368776113Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"2","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.369148891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"0","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.369261872Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"2","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.369841701Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:23:59.369877924Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.369554932Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37229)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.369679644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.370732981Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371268744Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[0]R flush_numbers[0..12]: [14, 14, 22, 16, 17, 23, 23, 23, 2, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371847713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[0]R generation[0..12]: [7, 7, 10, 8, 8, 10, 10, 10, 2, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371872335Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371888247Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[1]R flush_numbers[0..12]: [14, 14, 22, 16, 17, 23, 23, 23, 2, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371902868Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[1]R generation[0..12]: [7, 7, 10, 8, 8, 10, 10, 10, 2, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.37191647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371930461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[2]R flush_numbers[0..12]: [14, 14, 22, 16, 17, 23, 23, 23, 2, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371943883Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[2]R generation[0..12]: [7, 7, 10, 8, 8, 10, 10, 10, 2, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371957204Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371970835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Max found gen is 12","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.371987867Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372000818Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.37201346Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372030061Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"":"downstairs","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372043363Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372055184Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"0","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372068185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"1","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372080826Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"client":"2","":"downstairs","session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372094048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372106099Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"0cc4acdd-bcac-43ca-b224-703d89c18117 is now active with session: a4e7cc82-b9c5-4789-87a7-5481296ef1b2","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:24:09.37211862Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372132091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238,"session_id":"a4e7cc82-b9c5-4789-87a7-5481296ef1b2"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:09.372143923Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 304 len 16384 data: 3 2 2 2 02/30 Read block 211 len 8192 03/30 Read block 277 len 12288 04/30 Write block 257 len 20480 data: 2 2 2 2 2 05/30 Read block 305 len 36864 06/30 Read block 388 len 8192 07/30 Write block 163 len 20480 data: 2 2 3 2 2 08/30 Write block 104 len 32768 data: 2 3 3 3 3 3 4 4 09/30 Read block 323 len 4096 10/30 Flush 11/30 Write block 461 len 32768 data: 3 3 3 3 3 3 4 4 12/30 Read block 191 len 8192 13/30 Read block 70 len 40960 14/30 Read block 458 len 28672 15/30 Write block 273 len 28672 data: 2 2 2 2 2 2 2 16/30 Read block 264 len 36864 17/30 Read block 91 len 40960 18/30 Write block 513 len 4096 data: 3 19/30 Write block 334 len 40960 data: 4 4 4 3 4 4 4 3 3 3 20/30 Read block 494 len 12288 21/30 Read block 16 len 8192 22/30 Read block 173 len 4096 23/30 Write block 404 len 16384 data: 2 3 3 3 24/30 Write block 590 len 20480 data: 3 3 2 2 2 25/30 Write block 576 len 36864 data: 2 2 2 2 2 2 3 3 3 26/30 Read block 457 len 24576 27/30 Read block 449 len 20480 28/30 Read block 203 len 4096 29/30 Write block 392 len 8192 data: 2 2 30/30 Write block 385 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 8 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:09.391912586Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 14 14 14 F F F 1 020-039 7 7 7 14 14 14 F F F 2 040-059 10 10 10 22 22 22 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 8 8 8 17 17 17 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 10 10 10 23 23 23 F F F 7 140-159 10 10 10 23 23 23 F F F 8 160-179 12 12 12 25 25 25 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 12 12 12 25 25 25 F F F 13 260-279 12 12 12 25 25 25 T T T 14 280-299 11 11 11 24 24 24 F F F 15 300-319 12 12 12 25 25 25 F F F 16 320-339 11 11 11 24 24 24 T T T 17 340-359 11 11 11 24 24 24 T T T 18 360-379 5 5 5 10 10 10 F F F 19 380-399 6 6 6 11 11 11 T T T 20 400-419 10 10 10 23 23 23 T T T 21 420-439 10 10 10 20 20 20 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 6 6 6 11 11 11 T T T 24 480-499 4 4 4 8 8 8 F F F 25 500-519 8 8 8 17 17 17 T T T 26 520-539 9 9 9 19 19 19 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 1 1 1 1 1 1 T T T 29 580-599 9 9 9 19 19 19 T T T Max gen: 12, Max flush: 25 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 14 14 14 F F F 1 020-039 7 7 7 14 14 14 F F F 2 040-059 10 10 10 22 22 22 F F F 3 060-079 8 8 8 16 16 16 F F F 4 080-099 8 8 8 17 17 17 F F F 5 100-119 12 12 12 25 25 25 F F F 6 120-139 10 10 10 23 23 23 F F F 7 140-159 10 10 10 23 23 23 F F F 8 160-179 12 12 12 25 25 25 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 12 12 12 25 25 25 F F F 13 260-279 13 13 13 26 26 26 F F F 14 280-299 11 11 11 24 24 24 F F F 15 300-319 12 12 12 25 25 25 F F F 16 320-339 13 13 13 26 26 26 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 13 13 13 26 26 26 F F F 20 400-419 13 13 13 26 26 26 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 13 13 13 26 26 26 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 13 13 13 26 26 26 F F F 26 520-539 9 9 9 19 19 19 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 13 13 13 26 26 26 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 13, Max flush: 26 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.657029424Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.702937724Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.703780129Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.703826223Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.703843435Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.703860687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246} {"msg":"Crucible f5282cbc-2934-4669-b46e-6e2a94e8944f has session id: d7647dcb-13b8-4f54-b147-3786e83ac15a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.70448175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Upstairs opts: Upstairs UUID: f5282cbc-2934-4669-b46e-6e2a94e8944f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.704515673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Crucible stats registered with UUID: f5282cbc-2934-4669-b46e-6e2a94e8944f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.704533595Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:23.704553137Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705115164Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705151227Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246} {"msg":"f5282cbc-2934-4669-b46e-6e2a94e8944f active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705229705Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705247297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"0","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.70527675Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"0","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705303112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"1","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705330535Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"1","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705356488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"2","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705382691Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"2","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705407823Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705434716Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705464359Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51763)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705539886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55457)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.70557937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43582)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.705612114Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.706380441Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"0","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.706426176Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"1","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.706575511Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"2","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.707137928Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"1","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.70716615Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"2","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.707181872Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"0","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.707723647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.707825547Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[0]R flush_numbers[0..12]: [14, 14, 22, 16, 17, 25, 23, 23, 25, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.707977062Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[0]R generation[0..12]: [7, 7, 10, 8, 8, 12, 10, 10, 12, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.707995134Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708009395Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[1]R flush_numbers[0..12]: [14, 14, 22, 16, 17, 25, 23, 23, 25, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708022967Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[1]R generation[0..12]: [7, 7, 10, 8, 8, 12, 10, 10, 12, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708036028Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.70805371Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[2]R flush_numbers[0..12]: [14, 14, 22, 16, 17, 25, 23, 23, 25, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708067491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[2]R generation[0..12]: [7, 7, 10, 8, 8, 12, 10, 10, 12, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708080893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708094784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708107375Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708119987Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708138609Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.7081512Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"":"downstairs","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708164001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708175752Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"0","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708188603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"1","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708201585Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"client":"2","":"downstairs","session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708214416Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708226427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"f5282cbc-2934-4669-b46e-6e2a94e8944f is now active with session: d7647dcb-13b8-4f54-b147-3786e83ac15a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708238948Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.70825065Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246,"session_id":"d7647dcb-13b8-4f54-b147-3786e83ac15a"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:23.708264341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 93 len 4096 data: 3 02/30 Write block 509 len 4096 data: 3 03/30 Write block 322 len 20480 data: 3 3 2 2 2 04/30 Write block 171 len 24576 data: 2 2 2 2 2 2 05/30 Read block 458 len 4096 06/30 Read block 529 len 32768 07/30 Read block 369 len 8192 08/30 Write block 251 len 16384 data: 3 3 3 3 09/30 Write block 278 len 8192 data: 3 3 10/30 Write block 105 len 4096 data: 4 11/30 Read block 126 len 40960 12/30 Read block 53 len 8192 13/30 Flush 14/30 Write block 383 len 36864 data: 3 3 4 3 3 3 3 2 2 15/30 Write block 42 len 24576 data: 2 2 2 3 3 3 16/30 Read block 289 len 36864 17/30 Write block 79 len 36864 data: 5 5 4 2 2 2 2 4 4 18/30 Write block 317 len 28672 data: 2 3 3 3 3 4 4 19/30 Read block 99 len 40960 20/30 Write block 18 len 24576 data: 3 3 3 3 3 3 21/30 Write block 43 len 4096 data: 3 22/30 Read block 76 len 36864 23/30 Write block 1 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Read block 241 len 8192 25/30 Read block 273 len 28672 26/30 Read block 514 len 20480 27/30 Read block 426 len 12288 28/30 Read block 285 len 36864 29/30 Read block 441 len 32768 30/30 Read block 437 len 36864 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 16 17 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:23.727330974Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 14 14 14 T T T 1 020-039 7 7 7 14 14 14 T T T 2 040-059 10 10 10 22 22 22 T T T 3 060-079 8 8 8 16 16 16 T T T 4 080-099 14 14 14 27 27 27 T T T 5 100-119 14 14 14 27 27 27 F F F 6 120-139 10 10 10 23 23 23 F F F 7 140-159 10 10 10 23 23 23 F F F 8 160-179 14 14 14 27 27 27 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 14 14 14 27 27 27 F F F 13 260-279 14 14 14 27 27 27 F F F 14 280-299 11 11 11 24 24 24 F F F 15 300-319 12 12 12 25 25 25 T T T 16 320-339 14 14 14 27 27 27 T T T 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 13 13 13 26 26 26 T T T 20 400-419 13 13 13 26 26 26 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 13 13 13 26 26 26 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 14 14 14 27 27 27 F F F 26 520-539 9 9 9 19 19 19 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 13 13 13 26 26 26 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 14, Max flush: 27 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 28 28 28 F F F 1 020-039 15 15 15 28 28 28 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 10 10 10 23 23 23 F F F 7 140-159 10 10 10 23 23 23 F F F 8 160-179 14 14 14 27 27 27 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 9 9 9 19 19 19 F F F 12 240-259 14 14 14 27 27 27 F F F 13 260-279 14 14 14 27 27 27 F F F 14 280-299 11 11 11 24 24 24 F F F 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 13 13 13 26 26 26 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 13 13 13 26 26 26 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 14 14 14 27 27 27 F F F 26 520-539 9 9 9 19 19 19 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 13 13 13 26 26 26 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 15, Max flush: 28 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:27.982351382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:28.025502365Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.025941709Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.057784901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.058536726Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.05857587Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.058591152Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.058605543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254} {"msg":"Crucible 037db6a1-f139-45a1-a607-914bc9efdb73 has session id: 06dfb0da-25c3-4087-9024-f93aeed966dd","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059139237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Upstairs opts: Upstairs UUID: 037db6a1-f139-45a1-a607-914bc9efdb73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059220165Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Crucible stats registered with UUID: 037db6a1-f139-45a1-a607-914bc9efdb73","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059247688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:28.059262019Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059641628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059672421Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254} {"msg":"037db6a1-f139-45a1-a607-914bc9efdb73 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059721346Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059745448Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"0","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059769121Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"0","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059783132Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"1","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059798504Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"1","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059814985Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"2","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059829067Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"2","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059841828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059855249Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.059870211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38770)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.05995608Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:34743)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.060015146Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47139)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.060045898Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.060901645Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"1","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.060939469Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"2","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.06105206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"0","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.061704756Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"1","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.061734659Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"2","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.061752231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"0","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062388595Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062494165Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[0]R flush_numbers[0..12]: [28, 28, 28, 28, 28, 27, 23, 23, 27, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062565633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[0]R generation[0..12]: [15, 15, 15, 15, 15, 14, 10, 10, 14, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062593815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062609307Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[1]R flush_numbers[0..12]: [28, 28, 28, 28, 28, 27, 23, 23, 27, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062623329Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[1]R generation[0..12]: [15, 15, 15, 15, 15, 14, 10, 10, 14, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.06263788Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062652361Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[2]R flush_numbers[0..12]: [28, 28, 28, 28, 28, 27, 23, 23, 27, 14, 14, 19]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062666083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[2]R generation[0..12]: [15, 15, 15, 15, 15, 14, 10, 10, 14, 7, 7, 9]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062679554Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062693225Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062705667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-02-27T17:24:28.062725929Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.06273926Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062756112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"":"downstairs","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062769883Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062781834Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"0","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062795806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"1","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062808917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"client":"2","":"downstairs","session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062822038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.06283417Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"037db6a1-f139-45a1-a607-914bc9efdb73 is now active with session: 06dfb0da-25c3-4087-9024-f93aeed966dd","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062846811Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062858472Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254,"session_id":"06dfb0da-25c3-4087-9024-f93aeed966dd"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:28.062870414Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 513 len 16384 data: 4 2 2 2 02/30 Write block 156 len 12288 data: 2 2 2 03/30 Read block 288 len 20480 04/30 Read block 532 len 40960 05/30 Read block 534 len 40960 06/30 Write block 538 len 28672 data: 2 2 2 3 3 3 3 07/30 Read block 401 len 16384 08/30 Write block 518 len 8192 data: 2 2 09/30 Read block 73 len 24576 10/30 Write block 465 len 28672 data: 4 4 5 5 5 5 4 11/30 Read block 52 len 28672 12/30 Write block 3 len 8192 data: 3 3 13/30 Read block 181 len 16384 14/30 Read block 62 len 20480 15/30 Read block 493 len 12288 16/30 Write block 232 len 32768 data: 2 2 2 2 3 3 3 3 17/30 Read block 1 len 36864 18/30 Write block 517 len 24576 data: 2 3 3 2 2 2 19/30 Write block 529 len 32768 data: 4 4 4 2 2 2 2 2 20/30 Write block 268 len 16384 data: 3 3 3 2 21/30 Read block 227 len 24576 22/30 Read block 321 len 20480 23/30 Read block 517 len 16384 24/30 Write block 70 len 32768 data: 3 4 3 3 3 4 5 4 25/30 Write block 168 len 36864 data: 2 2 2 3 3 3 3 3 3 26/30 Read block 342 len 36864 27/30 Write block 488 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Read block 332 len 36864 29/30 Read block 499 len 40960 30/30 Write block 15 len 20480 data: 3 3 3 4 4 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:28.084142539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 28 28 28 T T T 1 020-039 15 15 15 28 28 28 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 15 15 15 28 28 28 T T T 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 10 10 10 23 23 23 F F F 7 140-159 10 10 10 23 23 23 T T T 8 160-179 14 14 14 27 27 27 T T T 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 9 9 9 19 19 19 T T T 12 240-259 14 14 14 27 27 27 F F F 13 260-279 14 14 14 27 27 27 T T T 14 280-299 11 11 11 24 24 24 F F F 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 13 13 13 26 26 26 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 13 13 13 26 26 26 T T T 24 480-499 4 4 4 8 8 8 T T T 25 500-519 14 14 14 27 27 27 T T T 26 520-539 9 9 9 19 19 19 T T T 27 540-559 10 10 10 22 22 22 T T T 28 560-579 13 13 13 26 26 26 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 15, Max flush: 28 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 F F F 1 020-039 15 15 15 28 28 28 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 16 16 16 29 29 29 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 10 10 10 23 23 23 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 14 14 14 27 27 27 F F F 13 260-279 16 16 16 29 29 29 F F F 14 280-299 11 11 11 24 24 24 F F F 15 300-319 15 15 15 28 28 28 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 13 13 13 26 26 26 F F F 21 420-439 10 10 10 20 20 20 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 16 16 16 29 29 29 F F F 26 520-539 16 16 16 29 29 29 F F F 27 540-559 16 16 16 29 29 29 F F F 28 560-579 13 13 13 26 26 26 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 16, Max flush: 29 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.390887564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:32.434140757Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.434641197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.466229443Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.467057327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.46709256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.467106172Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.467120233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262} {"msg":"Crucible 88a77f1b-baef-4fc0-864b-adab42b5483b has session id: b09d30e2-e3a4-4c80-8a3d-708aac59d0a6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.467647306Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Upstairs opts: Upstairs UUID: 88a77f1b-baef-4fc0-864b-adab42b5483b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.467697551Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Crucible stats registered with UUID: 88a77f1b-baef-4fc0-864b-adab42b5483b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.467718613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:32.467738095Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468202152Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468234015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262} {"msg":"88a77f1b-baef-4fc0-864b-adab42b5483b active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468288501Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468314243Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"0","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468337356Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"0","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468351287Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"1","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468373209Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"1","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468397572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"2","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468414553Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"2","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468427645Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468441506Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468461758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51208)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468588401Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63124)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468626675Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42769)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.468822475Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.469292612Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"2","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.469329416Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"1","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.469487502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"0","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.46996969Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"1","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.469996913Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"2","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470252319Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"0","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470602444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470647399Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[0]R flush_numbers[0..12]: [29, 28, 28, 29, 28, 27, 23, 29, 29, 14, 14, 29]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470795904Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[0]R generation[0..12]: [16, 15, 15, 16, 15, 14, 10, 16, 16, 7, 7, 16]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470816016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470833937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[1]R flush_numbers[0..12]: [29, 28, 28, 29, 28, 27, 23, 29, 29, 14, 14, 29]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470848039Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[1]R generation[0..12]: [16, 15, 15, 16, 15, 14, 10, 16, 16, 7, 7, 16]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.4708615Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470875192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[2]R flush_numbers[0..12]: [29, 28, 28, 29, 28, 27, 23, 29, 29, 14, 14, 29]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470888553Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[2]R generation[0..12]: [16, 15, 15, 16, 15, 14, 10, 16, 16, 7, 7, 16]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470901684Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470915196Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Max found gen is 17","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470927617Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 18 >= found:17","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470949799Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470974752Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.470998944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"":"downstairs","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.471024657Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.471048599Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"0","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.471074382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"1","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.471100775Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"client":"2","":"downstairs","session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.471130258Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.471144859Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"88a77f1b-baef-4fc0-864b-adab42b5483b is now active with session: b09d30e2-e3a4-4c80-8a3d-708aac59d0a6","v":0,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2025-02-27T17:24:32.47116137Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.471173592Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262,"session_id":"b09d30e2-e3a4-4c80-8a3d-708aac59d0a6"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:32.471185613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 522 len 20480 data: 3 3 3 3 3 02/30 Write block 248 len 24576 data: 3 3 4 4 4 4 03/30 Write block 434 len 28672 data: 2 2 2 2 2 3 2 04/30 Write block 28 len 12288 data: 4 4 2 05/30 Read block 521 len 40960 06/30 Read block 468 len 24576 07/30 Read block 553 len 24576 08/30 Flush 09/30 Write block 224 len 12288 data: 2 3 3 10/30 Write block 230 len 28672 data: 3 3 3 3 3 3 4 11/30 Write block 231 len 40960 data: 4 4 4 4 4 5 4 4 4 2 12/30 Write block 61 len 24576 data: 2 2 3 3 3 2 13/30 Read block 584 len 36864 14/30 Read block 475 len 12288 15/30 Write block 564 len 16384 data: 2 2 2 2 16/30 Read block 178 len 12288 17/30 Flush 18/30 Write block 132 len 24576 data: 2 2 2 2 2 2 19/30 Write block 298 len 28672 data: 2 2 3 3 3 3 4 20/30 Read block 323 len 20480 21/30 Read block 144 len 20480 22/30 Read block 146 len 40960 23/30 Flush 24/30 Write block 205 len 12288 data: 3 3 2 25/30 Read block 74 len 8192 26/30 Read block 79 len 20480 27/30 Read block 367 len 20480 28/30 Write block 267 len 16384 data: 3 4 4 4 29/30 Read block 248 len 40960 30/30 Write block 434 len 24576 data: 3 3 3 3 3 4 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:32.491409043Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 F F F 1 020-039 18 18 18 30 30 30 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 18 18 18 31 31 31 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 18 18 18 32 32 32 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 7 7 7 14 14 14 T T T 11 220-239 18 18 18 31 31 31 F F F 12 240-259 18 18 18 31 31 31 F F F 13 260-279 16 16 16 29 29 29 T T T 14 280-299 18 18 18 32 32 32 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 13 13 13 26 26 26 F F F 21 420-439 18 18 18 30 30 30 T T T 22 440-459 18 18 18 30 30 30 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 16 16 16 29 29 29 F F F 26 520-539 18 18 18 30 30 30 F F F 27 540-559 16 16 16 29 29 29 F F F 28 560-579 18 18 18 31 31 31 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 18, Max flush: 32 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 F F F 1 020-039 18 18 18 30 30 30 F F F 2 040-059 15 15 15 28 28 28 F F F 3 060-079 18 18 18 31 31 31 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 18 18 18 32 32 32 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 7 7 7 14 14 14 F F F 10 200-219 19 19 19 33 33 33 F F F 11 220-239 18 18 18 31 31 31 F F F 12 240-259 18 18 18 31 31 31 F F F 13 260-279 19 19 19 33 33 33 F F F 14 280-299 18 18 18 32 32 32 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 13 13 13 26 26 26 F F F 21 420-439 19 19 19 33 33 33 F F F 22 440-459 18 18 18 30 30 30 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 16 16 16 29 29 29 F F F 26 520-539 18 18 18 30 30 30 F F F 27 540-559 16 16 16 29 29 29 F F F 28 560-579 18 18 18 31 31 31 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 19, Max flush: 33 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.756088035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.800964391Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.801894185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.80194026Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.801954931Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.801969293Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270} {"msg":"Crucible 14b5aa56-4db4-4153-9e86-a6a3aaeea6be has session id: b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.802476424Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Upstairs opts: Upstairs UUID: 14b5aa56-4db4-4153-9e86-a6a3aaeea6be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.802514517Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Crucible stats registered with UUID: 14b5aa56-4db4-4153-9e86-a6a3aaeea6be","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.80253736Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:36.802557432Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803008187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803046451Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270} {"msg":"14b5aa56-4db4-4153-9e86-a6a3aaeea6be active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803125029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803141841Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"0","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803165383Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"0","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803189376Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"1","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803215198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"1","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803244351Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"2","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803271124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"2","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803296847Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803324679Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803350012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36000)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803374454Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38810)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803398917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:60298)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.803423149Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.804147062Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"2","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.804172805Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"0","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.804879856Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"0","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.804926971Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"2","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.805580477Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.80561445Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.806087438Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"1","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.806620072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"1","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[0]R flush_numbers[0..12]: [29, 30, 28, 31, 28, 27, 32, 29, 29, 14, 33, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807159146Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[0]R generation[0..12]: [16, 18, 15, 18, 15, 14, 18, 16, 16, 7, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807180508Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.80719497Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[1]R flush_numbers[0..12]: [29, 30, 28, 31, 28, 27, 32, 29, 29, 14, 33, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807208521Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[1]R generation[0..12]: [16, 18, 15, 18, 15, 14, 18, 16, 16, 7, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807221652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807235284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[2]R flush_numbers[0..12]: [29, 30, 28, 31, 28, 27, 32, 29, 29, 14, 33, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807248725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[2]R generation[0..12]: [16, 18, 15, 18, 15, 14, 18, 16, 16, 7, 19, 18]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807261686Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807275408Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807287989Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807308551Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807321402Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807337214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"":"downstairs","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807350155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807361877Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"0","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807374828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"1","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807387579Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"client":"2","":"downstairs","session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.80740035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807412262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"14b5aa56-4db4-4153-9e86-a6a3aaeea6be is now active with session: b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807424653Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807436344Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270,"session_id":"b859c2eb-65fd-40d4-b73c-bbaa1c05c0fd"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:36.807448185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 408 len 12288 data: 4 4 4 02/30 Read block 125 len 16384 03/30 Write block 123 len 24576 data: 4 4 3 3 2 2 04/30 Read block 105 len 12288 05/30 Read block 571 len 12288 06/30 Write block 48 len 20480 data: 3 3 3 3 3 07/30 Read block 507 len 32768 08/30 Write block 219 len 32768 data: 3 3 3 3 3 3 4 4 09/30 Write block 179 len 40960 data: 2 2 2 2 2 3 3 3 2 2 10/30 Read block 275 len 28672 11/30 Flush 12/30 Write block 459 len 12288 data: 2 2 4 13/30 Read block 247 len 20480 14/30 Write block 249 len 24576 data: 4 5 5 5 5 4 15/30 Flush 16/30 Write block 15 len 40960 data: 4 4 4 5 5 4 4 4 4 3 17/30 Write block 209 len 36864 data: 2 2 2 2 2 2 3 3 3 18/30 Read block 249 len 20480 19/30 Write block 533 len 32768 data: 3 3 3 3 2 3 3 3 20/30 Read block 551 len 12288 21/30 Read block 593 len 16384 22/30 Read block 259 len 28672 23/30 Write block 542 len 36864 data: 4 4 4 3 3 3 3 3 2 24/30 Read block 403 len 16384 25/30 Read block 346 len 36864 26/30 Write block 222 len 28672 data: 4 4 4 5 5 3 3 27/30 Read block 300 len 12288 28/30 Read block 472 len 16384 29/30 Read block 6 len 16384 30/30 Write block 427 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 10 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:36.829722362Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 29 29 29 T T T 1 020-039 18 18 18 30 30 30 T T T 2 040-059 20 20 20 34 34 34 F F F 3 060-079 18 18 18 31 31 31 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 20 20 20 34 34 34 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 F F F 10 200-219 20 20 20 34 34 34 T T T 11 220-239 20 20 20 34 34 34 T T T 12 240-259 20 20 20 35 35 35 F F F 13 260-279 19 19 19 33 33 33 F F F 14 280-299 18 18 18 32 32 32 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 20 20 20 34 34 34 F F F 21 420-439 19 19 19 33 33 33 T T T 22 440-459 20 20 20 35 35 35 F F F 23 460-479 20 20 20 35 35 35 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 16 16 16 29 29 29 F F F 26 520-539 18 18 18 30 30 30 T T T 27 540-559 16 16 16 29 29 29 T T T 28 560-579 18 18 18 31 31 31 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 20, Max flush: 35 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 36 36 36 F F F 1 020-039 21 21 21 36 36 36 F F F 2 040-059 20 20 20 34 34 34 F F F 3 060-079 18 18 18 31 31 31 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 F F F 6 120-139 20 20 20 34 34 34 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 19 19 19 33 33 33 F F F 14 280-299 18 18 18 32 32 32 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 15 15 15 28 28 28 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 15 15 15 28 28 28 F F F 20 400-419 20 20 20 34 34 34 F F F 21 420-439 21 21 21 36 36 36 F F F 22 440-459 20 20 20 35 35 35 F F F 23 460-479 20 20 20 35 35 35 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 16 16 16 29 29 29 F F F 26 520-539 21 21 21 36 36 36 F F F 27 540-559 21 21 21 36 36 36 F F F 28 560-579 18 18 18 31 31 31 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 21, Max flush: 36 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.080156237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.124833963Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.125666237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.125706091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.125720372Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.125735054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278} {"msg":"Crucible b6c98a82-005e-4c2a-b78b-36727cbd5d6c has session id: 30f639bb-168e-444d-a08e-2d64a6c487d7","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.126360277Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Upstairs opts: Upstairs UUID: b6c98a82-005e-4c2a-b78b-36727cbd5d6c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.126402781Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Crucible stats registered with UUID: b6c98a82-005e-4c2a-b78b-36727cbd5d6c","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.126428244Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:41.126441515Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.126943365Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.126974999Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278} {"msg":"b6c98a82-005e-4c2a-b78b-36727cbd5d6c active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127015813Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127036085Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"0","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127066808Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"0","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.12708757Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127109542Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127129254Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"2","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127150686Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"2","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127170608Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127191061Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127211543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53735)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127310202Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53803)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.12738352Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:41.127470749Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:41.127501202Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:41.127533285Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.127592131Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.128112513Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"2","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.128139556Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"0","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.12877601Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"0","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.128813404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"2","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.129331806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:41.129414635Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.128936467Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33276)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.129148668Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.130076302Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.130602155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"[0]R flush_numbers[0..12]: [36, 36, 34, 31, 28, 27, 34, 29, 34, 34, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131213726Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"[0]R generation[0..12]: [21, 21, 20, 18, 15, 14, 20, 16, 20, 20, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.13124805Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131267671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"[1]R flush_numbers[0..12]: [36, 36, 34, 31, 28, 27, 34, 29, 34, 34, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131284773Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"[1]R generation[0..12]: [21, 21, 20, 18, 15, 14, 20, 16, 20, 20, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131301275Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131318107Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"[2]R flush_numbers[0..12]: [36, 36, 34, 31, 28, 27, 34, 29, 34, 34, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131334518Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"[2]R generation[0..12]: [21, 21, 20, 18, 15, 14, 20, 16, 20, 20, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.13135147Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131372422Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131388384Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131404445Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131420007Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131435628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"":"downstairs","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.13145218Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131466892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"0","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131483263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"1","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131501635Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"client":"2","":"downstairs","session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131517807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131536449Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"b6c98a82-005e-4c2a-b78b-36727cbd5d6c is now active with session: 30f639bb-168e-444d-a08e-2d64a6c487d7","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.13155204Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131566672Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278,"session_id":"30f639bb-168e-444d-a08e-2d64a6c487d7"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:51.131581453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 406 len 16384 data: 4 4 5 5 02/30 Write block 385 len 16384 data: 5 4 4 4 03/30 Read block 107 len 40960 04/30 Flush 05/30 Write block 483 len 28672 data: 2 2 2 2 2 3 3 06/30 Read block 312 len 20480 07/30 Read block 90 len 32768 08/30 Write block 116 len 4096 data: 4 09/30 Read block 71 len 8192 10/30 Read block 87 len 20480 11/30 Read block 144 len 28672 12/30 Read block 586 len 8192 13/30 Write block 436 len 32768 data: 4 4 4 5 3 2 2 2 14/30 Read block 531 len 16384 15/30 Read block 103 len 8192 16/30 Read block 531 len 16384 17/30 Read block 228 len 12288 18/30 Read block 511 len 32768 19/30 Read block 508 len 20480 20/30 Read block 99 len 12288 21/30 Read block 581 len 28672 22/30 Write block 267 len 4096 data: 4 23/30 Write block 16 len 20480 data: 5 5 6 6 5 24/30 Read block 222 len 40960 25/30 Write block 115 len 28672 data: 4 5 7 7 7 6 5 26/30 Read block 92 len 28672 27/30 Write block 31 len 40960 data: 2 2 2 2 2 2 2 2 2 2 28/30 Write block 330 len 28672 data: 3 3 4 4 5 5 5 29/30 Write block 59 len 12288 data: 2 2 3 30/30 Read block 28 len 36864 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 7 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 25 25 26 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:51.151860079Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 36 36 36 T T T 1 020-039 21 21 21 36 36 36 T T T 2 040-059 20 20 20 34 34 34 T T T 3 060-079 18 18 18 31 31 31 T T T 4 080-099 15 15 15 28 28 28 F F F 5 100-119 14 14 14 27 27 27 T T T 6 120-139 20 20 20 34 34 34 T T T 7 140-159 16 16 16 29 29 29 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 19 19 19 33 33 33 T T T 14 280-299 18 18 18 32 32 32 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 15 15 15 28 28 28 T T T 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 22 22 22 37 37 37 F F F 20 400-419 22 22 22 37 37 37 F F F 21 420-439 21 21 21 36 36 36 T T T 22 440-459 20 20 20 35 35 35 T T T 23 460-479 20 20 20 35 35 35 F F F 24 480-499 16 16 16 29 29 29 T T T 25 500-519 16 16 16 29 29 29 F F F 26 520-539 21 21 21 36 36 36 F F F 27 540-559 21 21 21 36 36 36 F F F 28 560-579 18 18 18 31 31 31 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 22, Max flush: 37 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 23 23 23 38 38 38 F F F 2 040-059 23 23 23 38 38 38 F F F 3 060-079 23 23 23 38 38 38 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 23 23 23 38 38 38 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 18 18 18 32 32 32 F F F 15 300-319 18 18 18 32 32 32 F F F 16 320-339 23 23 23 38 38 38 F F F 17 340-359 13 13 13 26 26 26 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 22 22 22 37 37 37 F F F 20 400-419 22 22 22 37 37 37 F F F 21 420-439 23 23 23 38 38 38 F F F 22 440-459 23 23 23 38 38 38 F F F 23 460-479 20 20 20 35 35 35 F F F 24 480-499 23 23 23 38 38 38 F F F 25 500-519 16 16 16 29 29 29 F F F 26 520-539 21 21 21 36 36 36 F F F 27 540-559 21 21 21 36 36 36 F F F 28 560-579 18 18 18 31 31 31 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 23, Max flush: 38 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.426629669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.471488844Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.472334959Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.472376413Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.472392475Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.472406966Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286} {"msg":"Crucible 8d094722-0c0c-44b4-9bc6-6ebd03f44092 has session id: 97297b15-fecc-4b00-abba-2679a22ddb5e","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.472948301Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Upstairs opts: Upstairs UUID: 8d094722-0c0c-44b4-9bc6-6ebd03f44092, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473059192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Crucible stats registered with UUID: 8d094722-0c0c-44b4-9bc6-6ebd03f44092","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473094375Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:55.473111327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.47354034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473563703Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286} {"msg":"8d094722-0c0c-44b4-9bc6-6ebd03f44092 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473640921Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473658472Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"0","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473684515Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"0","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473709077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"1","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.47373604Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"1","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473762483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"2","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473788416Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"2","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473811778Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.47383759Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473863133Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64578)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473890236Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44576)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473916218Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49003)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.473960753Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.474616569Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"0","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.474644532Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"2","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.475332341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"0","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.475372275Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"2","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.476053524Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.476079327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"1","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.476094448Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.476658115Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"1","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[0]R flush_numbers[0..12]: [38, 38, 38, 38, 28, 38, 38, 29, 34, 34, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477189628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[0]R generation[0..12]: [23, 23, 23, 23, 15, 23, 23, 16, 20, 20, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477214391Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477229152Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[1]R flush_numbers[0..12]: [38, 38, 38, 38, 28, 38, 38, 29, 34, 34, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477242654Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[1]R generation[0..12]: [23, 23, 23, 23, 15, 23, 23, 16, 20, 20, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477256155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477269797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[2]R flush_numbers[0..12]: [38, 38, 38, 38, 28, 38, 38, 29, 34, 34, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477282948Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[2]R generation[0..12]: [23, 23, 23, 23, 15, 23, 23, 16, 20, 20, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477296429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-02-27T17:24:55.477313551Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477333383Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477346614Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477359286Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477371747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"":"downstairs","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477384988Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477396619Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"0","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477409851Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"1","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477422802Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"client":"2","":"downstairs","session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477435913Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477447794Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"8d094722-0c0c-44b4-9bc6-6ebd03f44092 is now active with session: 97297b15-fecc-4b00-abba-2679a22ddb5e","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477460296Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477471737Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286,"session_id":"97297b15-fecc-4b00-abba-2679a22ddb5e"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:55.477483518Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 336 len 32768 data: 6 4 5 5 5 4 4 4 03/30 Read block 127 len 8192 04/30 Read block 222 len 12288 05/30 Read block 191 len 36864 06/30 Read block 558 len 32768 07/30 Read block 285 len 20480 08/30 Write block 291 len 40960 data: 3 3 2 3 3 3 3 3 3 4 09/30 Write block 54 len 32768 data: 3 4 4 4 4 3 3 4 10/30 Read block 27 len 16384 11/30 Read block 134 len 20480 12/30 Read block 427 len 20480 13/30 Write block 572 len 40960 data: 2 2 2 2 3 3 3 3 3 3 14/30 Read block 302 len 4096 15/30 Write block 541 len 8192 data: 4 5 16/30 Flush 17/30 Write block 193 len 12288 data: 2 2 2 18/30 Write block 509 len 8192 data: 4 3 19/30 Read block 592 len 8192 20/30 Write block 293 len 24576 data: 3 4 4 4 4 4 21/30 Read block 461 len 12288 22/30 Write block 304 len 36864 data: 5 3 3 3 2 3 3 3 3 23/30 Write block 512 len 36864 data: 3 5 3 3 3 3 4 4 3 24/30 Read block 297 len 32768 25/30 Write block 120 len 8192 data: 7 6 26/30 Write block 545 len 28672 data: 4 4 4 4 4 3 2 27/30 Read block 215 len 40960 28/30 Read block 416 len 16384 29/30 Read block 336 len 8192 30/30 Read block 417 len 16384 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:55.498347922Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 23 23 23 38 38 38 F F F 2 040-059 24 24 24 40 40 40 F F F 3 060-079 24 24 24 40 40 40 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 23 23 23 38 38 38 T T T 7 140-159 16 16 16 29 29 29 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 20 20 20 34 34 34 T T T 10 200-219 21 21 21 36 36 36 F F F 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 24 24 24 40 40 40 T T T 15 300-319 24 24 24 40 40 40 T T T 16 320-339 24 24 24 40 40 40 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 22 22 22 37 37 37 F F F 20 400-419 22 22 22 37 37 37 F F F 21 420-439 23 23 23 38 38 38 F F F 22 440-459 23 23 23 38 38 38 F F F 23 460-479 20 20 20 35 35 35 F F F 24 480-499 23 23 23 38 38 38 F F F 25 500-519 16 16 16 29 29 29 T T T 26 520-539 21 21 21 36 36 36 T T T 27 540-559 24 24 24 40 40 40 T T T 28 560-579 24 24 24 40 40 40 F F F 29 580-599 24 24 24 40 40 40 F F F Max gen: 24, Max flush: 40 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 F F F 1 020-039 23 23 23 38 38 38 F F F 2 040-059 24 24 24 40 40 40 F F F 3 060-079 24 24 24 40 40 40 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 25 25 25 41 41 41 F F F 7 140-159 16 16 16 29 29 29 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 25 25 25 41 41 41 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 25 25 25 41 41 41 F F F 15 300-319 25 25 25 41 41 41 F F F 16 320-339 24 24 24 40 40 40 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 22 22 22 37 37 37 F F F 20 400-419 22 22 22 37 37 37 F F F 21 420-439 23 23 23 38 38 38 F F F 22 440-459 23 23 23 38 38 38 F F F 23 460-479 20 20 20 35 35 35 F F F 24 480-499 23 23 23 38 38 38 F F F 25 500-519 25 25 25 41 41 41 F F F 26 520-539 25 25 25 41 41 41 F F F 27 540-559 25 25 25 41 41 41 F F F 28 560-579 24 24 24 40 40 40 F F F 29 580-599 24 24 24 40 40 40 F F F Max gen: 25, Max flush: 41 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.756991736Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.80462332Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.805449443Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.805494118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.805508939Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.80552355Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294} {"msg":"Crucible 106604b7-4911-467f-8c0c-91c40c779ba2 has session id: d10d7a75-c02d-4c85-a052-e354f3d54ded","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.806226551Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Upstairs opts: Upstairs UUID: 106604b7-4911-467f-8c0c-91c40c779ba2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.806278037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Crucible stats registered with UUID: 106604b7-4911-467f-8c0c-91c40c779ba2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.806304269Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:59.806322431Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.806870586Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.806894589Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294} {"msg":"106604b7-4911-467f-8c0c-91c40c779ba2 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.806964596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.806979747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"0","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.80700317Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"0","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807017221Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"1","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807031602Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"1","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807044514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"2","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807058515Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"2","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807078387Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807149594Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807181228Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"ds_connection connected from Ok(127.0.0.1:50402)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807287328Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"ds_connection connected from Ok(127.0.0.1:34412)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807319842Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"ds_connection connected from Ok(127.0.0.1:51756)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.807427413Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.808132944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"0","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.808161116Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"1","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.808666427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"2","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.808798731Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"0","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.808938605Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"1","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.80929201Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"2","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809496141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.80958707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[0]R flush_numbers[0..12]: [38, 38, 40, 40, 28, 38, 41, 29, 34, 41, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809819324Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[0]R generation[0..12]: [23, 23, 24, 24, 15, 23, 25, 16, 20, 25, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809843146Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809858448Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[1]R flush_numbers[0..12]: [38, 38, 40, 40, 28, 38, 41, 29, 34, 41, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809876219Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[1]R generation[0..12]: [23, 23, 24, 24, 15, 23, 25, 16, 20, 25, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809889991Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809903702Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[2]R flush_numbers[0..12]: [38, 38, 40, 40, 28, 38, 41, 29, 34, 41, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809916844Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[2]R generation[0..12]: [23, 23, 24, 24, 15, 23, 25, 16, 20, 25, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809929955Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809943446Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809955937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809968649Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.809987841Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810000582Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"":"downstairs","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810013483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810025235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"0","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810038336Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"1","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810051247Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"client":"2","":"downstairs","session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810066499Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.81007843Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"106604b7-4911-467f-8c0c-91c40c779ba2 is now active with session: d10d7a75-c02d-4c85-a052-e354f3d54ded","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810091181Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810103072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294,"session_id":"d10d7a75-c02d-4c85-a052-e354f3d54ded"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:24:59.810114944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 505 len 16384 data: 2 3 2 2 02/30 Read block 308 len 12288 03/30 Read block 334 len 40960 04/30 Read block 559 len 8192 05/30 Write block 390 len 32768 data: 3 3 3 3 2 2 2 2 06/30 Write block 550 len 4096 data: 4 07/30 Read block 230 len 32768 08/30 Write block 73 len 12288 data: 4 4 5 09/30 Flush 10/30 Write block 149 len 8192 data: 3 2 11/30 Read block 91 len 40960 12/30 Read block 512 len 4096 13/30 Flush 14/30 Write block 8 len 36864 data: 3 3 3 2 2 2 2 5 6 15/30 Read block 67 len 8192 16/30 Read block 161 len 8192 17/30 Read block 102 len 4096 18/30 Read block 67 len 8192 19/30 Read block 558 len 12288 20/30 Read block 459 len 8192 21/30 Read block 198 len 20480 22/30 Read block 222 len 4096 23/30 Read block 226 len 16384 24/30 Write block 332 len 16384 data: 5 5 6 6 25/30 Read block 123 len 16384 26/30 Read block 360 len 12288 27/30 Read block 315 len 12288 28/30 Write block 147 len 8192 data: 2 2 29/30 Write block 565 len 20480 data: 3 3 3 2 2 30/30 Write block 125 len 40960 data: 4 4 3 3 2 2 2 3 3 3 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 16 16 15 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:24:59.829704739Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 38 38 38 T T T 1 020-039 23 23 23 38 38 38 F F F 2 040-059 24 24 24 40 40 40 F F F 3 060-079 26 26 26 42 42 42 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 25 25 25 41 41 41 T T T 7 140-159 26 26 26 43 43 43 T T T 8 160-179 20 20 20 34 34 34 F F F 9 180-199 25 25 25 41 41 41 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 25 25 25 41 41 41 F F F 15 300-319 25 25 25 41 41 41 F F F 16 320-339 24 24 24 40 40 40 T T T 17 340-359 24 24 24 40 40 40 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 26 26 26 42 42 42 F F F 20 400-419 22 22 22 37 37 37 F F F 21 420-439 23 23 23 38 38 38 F F F 22 440-459 23 23 23 38 38 38 F F F 23 460-479 20 20 20 35 35 35 F F F 24 480-499 23 23 23 38 38 38 F F F 25 500-519 26 26 26 42 42 42 F F F 26 520-539 25 25 25 41 41 41 F F F 27 540-559 26 26 26 42 42 42 F F F 28 560-579 24 24 24 40 40 40 T T T 29 580-599 24 24 24 40 40 40 F F F Max gen: 26, Max flush: 43 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 44 44 44 F F F 1 020-039 23 23 23 38 38 38 F F F 2 040-059 24 24 24 40 40 40 F F F 3 060-079 26 26 26 42 42 42 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 27 27 27 44 44 44 F F F 7 140-159 27 27 27 44 44 44 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 25 25 25 41 41 41 F F F 10 200-219 21 21 21 36 36 36 F F F 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 25 25 25 41 41 41 F F F 15 300-319 25 25 25 41 41 41 F F F 16 320-339 27 27 27 44 44 44 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 5 5 5 10 10 10 F F F 19 380-399 26 26 26 42 42 42 F F F 20 400-419 22 22 22 37 37 37 F F F 21 420-439 23 23 23 38 38 38 F F F 22 440-459 23 23 23 38 38 38 F F F 23 460-479 20 20 20 35 35 35 F F F 24 480-499 23 23 23 38 38 38 F F F 25 500-519 26 26 26 42 42 42 F F F 26 520-539 25 25 25 41 41 41 F F F 27 540-559 26 26 26 42 42 42 F F F 28 560-579 27 27 27 44 44 44 F F F 29 580-599 24 24 24 40 40 40 F F F Max gen: 27, Max flush: 44 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.069315393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.113646434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.114367717Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.114412962Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.114426763Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.114458416Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302} {"msg":"Crucible 8b56f758-2ab7-4ab4-a419-241b9a8e2572 has session id: e0807f6e-bb9a-46b6-b2a1-2b96e0543c74","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115021753Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Upstairs opts: Upstairs UUID: 8b56f758-2ab7-4ab4-a419-241b9a8e2572, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115059747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Crucible stats registered with UUID: 8b56f758-2ab7-4ab4-a419-241b9a8e2572","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115081589Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:14.11509511Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115618423Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115643076Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302} {"msg":"8b56f758-2ab7-4ab4-a419-241b9a8e2572 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115710692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115727804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"0","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115749607Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"0","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115763198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"1","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.11578031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"1","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115793461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115807192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115819883Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115832955Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115846246Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"ds_connection connected from Ok(127.0.0.1:54179)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115945836Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"ds_connection connected from Ok(127.0.0.1:52596)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.115983Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:14.116014523Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:14.116042956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:14.116253207Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.116331725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.116744737Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"0","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.11677131Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"1","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.117357199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"1","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.117383571Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"0","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.118034967Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:14.1180595Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.117672751Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"ds_connection connected from Ok(127.0.0.1:48670)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.117803214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.119038758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.119673542Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[0]R flush_numbers[0..12]: [44, 38, 40, 42, 28, 38, 44, 44, 34, 41, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120281203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[0]R generation[0..12]: [27, 23, 24, 26, 15, 23, 27, 27, 20, 25, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120306056Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120321998Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[1]R flush_numbers[0..12]: [44, 38, 40, 42, 28, 38, 44, 44, 34, 41, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120336189Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[1]R generation[0..12]: [27, 23, 24, 26, 15, 23, 27, 27, 20, 25, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.12034971Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120363692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[2]R flush_numbers[0..12]: [44, 38, 40, 42, 28, 38, 44, 44, 34, 41, 36, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120381764Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[2]R generation[0..12]: [27, 23, 24, 26, 15, 23, 27, 27, 20, 25, 21, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120397525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120411197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120423908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120437089Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120455881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120468533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"":"downstairs","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120481434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120493055Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"0","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120505976Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"1","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120518698Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"client":"2","":"downstairs","session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120531549Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.12054326Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"8b56f758-2ab7-4ab4-a419-241b9a8e2572 is now active with session: e0807f6e-bb9a-46b6-b2a1-2b96e0543c74","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120555871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120567563Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302,"session_id":"e0807f6e-bb9a-46b6-b2a1-2b96e0543c74"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:24.120581554Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 302 len 36864 data: 4 4 6 4 4 4 3 4 4 03/30 Write block 524 len 40960 data: 4 4 4 3 3 5 5 5 3 4 04/30 Read block 461 len 36864 05/30 Read block 234 len 28672 06/30 Read block 245 len 32768 07/30 Read block 223 len 28672 08/30 Read block 215 len 36864 09/30 Write block 371 len 8192 data: 3 3 10/30 Write block 287 len 16384 data: 3 3 3 3 11/30 Read block 19 len 4096 12/30 Write block 2 len 28672 data: 3 4 4 3 3 3 4 13/30 Write block 134 len 40960 data: 4 3 3 3 2 3 3 3 3 3 14/30 Write block 202 len 36864 data: 4 3 3 4 4 3 2 3 3 15/30 Write block 136 len 40960 data: 4 4 3 4 4 4 4 4 2 2 16/30 Read block 217 len 32768 17/30 Write block 133 len 40960 data: 4 5 4 5 5 4 5 5 5 5 18/30 Read block 57 len 28672 19/30 Read block 213 len 20480 20/30 Read block 236 len 4096 21/30 Write block 508 len 20480 data: 3 5 4 3 4 22/30 Write block 131 len 24576 data: 3 4 5 6 5 6 23/30 Write block 408 len 36864 data: 6 6 5 2 4 4 3 2 2 24/30 Write block 37 len 20480 data: 3 3 3 3 2 25/30 Read block 172 len 8192 26/30 Read block 255 len 28672 27/30 Write block 460 len 28672 data: 3 5 4 4 4 5 5 28/30 Read block 184 len 8192 29/30 Read block 313 len 36864 30/30 Write block 464 len 36864 data: 5 6 6 6 6 6 6 5 3 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:24.142290623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 44 44 44 T T T 1 020-039 23 23 23 38 38 38 T T T 2 040-059 24 24 24 40 40 40 T T T 3 060-079 26 26 26 42 42 42 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 27 27 27 44 44 44 T T T 7 140-159 27 27 27 44 44 44 T T T 8 160-179 20 20 20 34 34 34 F F F 9 180-199 25 25 25 41 41 41 F F F 10 200-219 21 21 21 36 36 36 T T T 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 25 25 25 41 41 41 T T T 15 300-319 25 25 25 41 41 41 T T T 16 320-339 27 27 27 44 44 44 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 5 5 5 10 10 10 T T T 19 380-399 26 26 26 42 42 42 F F F 20 400-419 22 22 22 37 37 37 T T T 21 420-439 23 23 23 38 38 38 F F F 22 440-459 23 23 23 38 38 38 F F F 23 460-479 20 20 20 35 35 35 T T T 24 480-499 23 23 23 38 38 38 F F F 25 500-519 26 26 26 42 42 42 T T T 26 520-539 25 25 25 41 41 41 T T T 27 540-559 26 26 26 42 42 42 F F F 28 560-579 27 27 27 44 44 44 F F F 29 580-599 24 24 24 40 40 40 F F F Max gen: 27, Max flush: 44 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 28 28 28 45 45 45 F F F 2 040-059 28 28 28 45 45 45 F F F 3 060-079 26 26 26 42 42 42 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 28 28 28 45 45 45 F F F 7 140-159 28 28 28 45 45 45 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 25 25 25 41 41 41 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 21 21 21 36 36 36 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 28 28 28 45 45 45 F F F 15 300-319 28 28 28 45 45 45 F F F 16 320-339 27 27 27 44 44 44 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 28 28 28 45 45 45 F F F 19 380-399 26 26 26 42 42 42 F F F 20 400-419 28 28 28 45 45 45 F F F 21 420-439 23 23 23 38 38 38 F F F 22 440-459 23 23 23 38 38 38 F F F 23 460-479 28 28 28 45 45 45 F F F 24 480-499 23 23 23 38 38 38 F F F 25 500-519 28 28 28 45 45 45 F F F 26 520-539 28 28 28 45 45 45 F F F 27 540-559 26 26 26 42 42 42 F F F 28 560-579 27 27 27 44 44 44 F F F 29 580-599 24 24 24 40 40 40 F F F Max gen: 28, Max flush: 45 Begin loop 15 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.433666639Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.480522445Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.481230996Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.48127445Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.481289572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.481304673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310} {"msg":"Crucible 41003eac-53fc-458d-bf4b-eecd52b28ef4 has session id: 5ab3d164-a924-4918-b9b5-0659fdd4a950","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.481942488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Upstairs opts: Upstairs UUID: 41003eac-53fc-458d-bf4b-eecd52b28ef4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.481971371Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Crucible stats registered with UUID: 41003eac-53fc-458d-bf4b-eecd52b28ef4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.481993993Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:38.482018536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482580522Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482614816Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310} {"msg":"41003eac-53fc-458d-bf4b-eecd52b28ef4 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482675122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482700664Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"0","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482724037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"0","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482739678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"1","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.48275809Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"1","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482771581Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"2","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482785433Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"2","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482808085Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482835368Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482860721Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"ds_connection connected from Ok(127.0.0.1:50715)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.482973022Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"ds_connection connected from Ok(127.0.0.1:59351)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.483015096Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"ds_connection connected from Ok(127.0.0.1:63480)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.483043389Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.483707396Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"0","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.483734699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"1","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.484373163Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"1","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.484400066Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"0","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.485049131Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.485088445Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.485453742Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"2","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.485976315Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"2","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[0]R flush_numbers[0..12]: [45, 45, 45, 42, 28, 38, 45, 45, 34, 41, 45, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486508618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[0]R generation[0..12]: [28, 28, 28, 26, 15, 23, 28, 28, 20, 25, 28, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.48652738Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486541892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[1]R flush_numbers[0..12]: [45, 45, 45, 42, 28, 38, 45, 45, 34, 41, 45, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486555343Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[1]R generation[0..12]: [28, 28, 28, 26, 15, 23, 28, 28, 20, 25, 28, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486570915Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486585046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[2]R flush_numbers[0..12]: [45, 45, 45, 42, 28, 38, 45, 45, 34, 41, 45, 36]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486598327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[2]R generation[0..12]: [28, 28, 28, 26, 15, 23, 28, 28, 20, 25, 28, 21]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486611499Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.48662497Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Max found gen is 29","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486637702Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Generation requested: 30 >= found:29","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486656393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486672185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486685086Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"":"downstairs","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486698128Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486709669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"0","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.48672253Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"1","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486735251Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"client":"2","":"downstairs","session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486748013Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486759884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"41003eac-53fc-458d-bf4b-eecd52b28ef4 is now active with session: 5ab3d164-a924-4918-b9b5-0659fdd4a950","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486772245Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:25:38.486783956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310,"session_id":"5ab3d164-a924-4918-b9b5-0659fdd4a950"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:38.486797618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 415 len 4096 data: 3 02/30 Write block 62 len 28672 data: 3 4 4 4 3 3 3 03/30 Write block 388 len 36864 data: 5 4 4 4 4 4 3 3 3 04/30 Read block 473 len 28672 05/30 Read block 163 len 32768 06/30 Read block 179 len 16384 07/30 Write block 452 len 40960 data: 2 2 2 2 2 2 2 3 4 6 08/30 Flush 09/30 Write block 297 len 40960 data: 5 5 4 5 4 5 5 7 5 5 10/30 Read block 412 len 16384 11/30 Write block 35 len 36864 data: 3 3 4 4 4 4 3 3 4 12/30 Flush 13/30 Write block 54 len 16384 data: 4 5 5 5 14/30 Flush 15/30 Write block 557 len 12288 data: 2 2 3 16/30 Read block 477 len 28672 17/30 Flush 18/30 Write block 328 len 36864 data: 2 3 4 4 6 6 7 7 7 19/30 Write block 324 len 28672 data: 3 3 3 2 3 4 5 20/30 Read block 513 len 24576 21/30 Read block 27 len 40960 22/30 Read block 391 len 28672 23/30 Write block 235 len 8192 data: 5 6 24/30 Read block 78 len 28672 25/30 Read block 530 len 8192 26/30 Read block 516 len 32768 27/30 Write block 102 len 36864 data: 2 2 3 5 4 4 4 4 5 28/30 Write block 195 len 4096 data: 3 29/30 Write block 480 len 20480 data: 2 2 2 3 3 30/30 Read block 295 len 28672 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 9 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 12 13 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:38.507415567Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 30 30 30 47 47 47 F F F 2 040-059 30 30 30 48 48 48 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 23 23 23 38 38 38 T T T 6 120-139 28 28 28 45 45 45 F F F 7 140-159 28 28 28 45 45 45 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 25 25 25 41 41 41 T T T 10 200-219 28 28 28 45 45 45 F F F 11 220-239 21 21 21 36 36 36 T T T 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 27 27 27 44 44 44 T T T 17 340-359 24 24 24 40 40 40 F F F 18 360-379 28 28 28 45 45 45 F F F 19 380-399 30 30 30 46 46 46 F F F 20 400-419 30 30 30 46 46 46 F F F 21 420-439 23 23 23 38 38 38 F F F 22 440-459 30 30 30 46 46 46 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 23 23 23 38 38 38 T T T 25 500-519 28 28 28 45 45 45 F F F 26 520-539 28 28 28 45 45 45 F F F 27 540-559 30 30 30 49 49 49 F F F 28 560-579 27 27 27 44 44 44 F F F 29 580-599 24 24 24 40 40 40 F F F Max gen: 30, Max flush: 49 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 30 30 30 47 47 47 F F F 2 040-059 30 30 30 48 48 48 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 31 31 31 50 50 50 F F F 6 120-139 28 28 28 45 45 45 F F F 7 140-159 28 28 28 45 45 45 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 31 31 31 50 50 50 F F F 10 200-219 28 28 28 45 45 45 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 28 28 28 45 45 45 F F F 19 380-399 30 30 30 46 46 46 F F F 20 400-419 30 30 30 46 46 46 F F F 21 420-439 23 23 23 38 38 38 F F F 22 440-459 30 30 30 46 46 46 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 31 31 31 50 50 50 F F F 25 500-519 28 28 28 45 45 45 F F F 26 520-539 28 28 28 45 45 45 F F F 27 540-559 30 30 30 49 49 49 F F F 28 560-579 27 27 27 44 44 44 F F F 29 580-599 24 24 24 40 40 40 F F F Max gen: 31, Max flush: 50 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.751600353Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.796307951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.797334265Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.79738184Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.797397592Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.797412203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318} {"msg":"Crucible 56d4b8c9-d6d5-46de-8f78-a55fbfb53887 has session id: 01b6d713-5f10-44c6-a4e0-27f84ff63af4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798064259Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Upstairs opts: Upstairs UUID: 56d4b8c9-d6d5-46de-8f78-a55fbfb53887, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798108533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Crucible stats registered with UUID: 56d4b8c9-d6d5-46de-8f78-a55fbfb53887","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798129475Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:42.798158118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.79866906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798700723Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318} {"msg":"56d4b8c9-d6d5-46de-8f78-a55fbfb53887 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.79876559Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798791472Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"0","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798815275Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"0","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798830396Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"1","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798844947Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"1","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798861899Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"2","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798887922Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"2","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798912524Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798937627Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798962349Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42933)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.798992252Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"ds_connection connected from Ok(127.0.0.1:60070)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.799118165Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62696)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.799247058Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.799721316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"0","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.799758119Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"2","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.800347769Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"0","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.800505965Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"2","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.800947439Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.801071892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.801553341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"1","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802130539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"1","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[0]R flush_numbers[0..12]: [45, 47, 48, 46, 28, 50, 45, 45, 34, 50, 45, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802649031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[0]R generation[0..12]: [28, 30, 30, 30, 15, 31, 28, 28, 20, 31, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802668513Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802683214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[1]R flush_numbers[0..12]: [45, 47, 48, 46, 28, 50, 45, 45, 34, 50, 45, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802696676Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[1]R generation[0..12]: [28, 30, 30, 30, 15, 31, 28, 28, 20, 31, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802709907Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802723449Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[2]R flush_numbers[0..12]: [45, 47, 48, 46, 28, 50, 45, 45, 34, 50, 45, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802741371Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[2]R generation[0..12]: [28, 30, 30, 30, 15, 31, 28, 28, 20, 31, 28, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802755182Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802768933Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802781815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802802377Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802815028Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802827679Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"":"downstairs","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802840341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802852332Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"0","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802865303Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"1","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802877954Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"client":"2","":"downstairs","session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802890566Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802902367Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"56d4b8c9-d6d5-46de-8f78-a55fbfb53887 is now active with session: 01b6d713-5f10-44c6-a4e0-27f84ff63af4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802914878Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802926429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318,"session_id":"01b6d713-5f10-44c6-a4e0-27f84ff63af4"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:42.802940211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 199 len 16384 data: 3 3 4 5 02/30 Flush 03/30 Write block 382 len 20480 data: 2 4 4 6 5 04/30 Write block 452 len 28672 data: 3 3 3 3 3 3 3 05/30 Write block 495 len 4096 data: 3 06/30 Write block 418 len 8192 data: 2 2 07/30 Read block 239 len 12288 08/30 Read block 359 len 16384 09/30 Write block 438 len 28672 data: 5 6 4 3 3 3 2 10/30 Write block 575 len 4096 data: 3 11/30 Write block 385 len 40960 data: 7 6 5 6 5 5 5 5 5 4 12/30 Read block 186 len 12288 13/30 Read block 269 len 24576 14/30 Read block 364 len 36864 15/30 Read block 184 len 16384 16/30 Write block 252 len 32768 data: 6 6 5 3 2 3 3 3 17/30 Flush 18/30 Write block 538 len 28672 data: 4 4 4 5 6 5 5 19/30 Read block 378 len 40960 20/30 Read block 373 len 20480 21/30 Flush 22/30 Write block 184 len 8192 data: 4 4 23/30 Write block 551 len 4096 data: 3 24/30 Read block 475 len 40960 25/30 Flush 26/30 Flush 27/30 Write block 197 len 12288 data: 2 2 4 28/30 Write block 556 len 32768 data: 2 3 3 4 2 2 2 2 29/30 Write block 149 len 28672 data: 4 3 3 3 3 3 2 30/30 Read block 582 len 12288 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:42.822423596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 30 30 30 47 47 47 F F F 2 040-059 30 30 30 48 48 48 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 31 31 31 50 50 50 F F F 6 120-139 28 28 28 45 45 45 F F F 7 140-159 28 28 28 45 45 45 T T T 8 160-179 20 20 20 34 34 34 F F F 9 180-199 32 32 32 54 54 54 T T T 10 200-219 32 32 32 51 51 51 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 28 28 28 45 45 45 F F F 19 380-399 32 32 32 52 52 52 F F F 20 400-419 32 32 32 52 52 52 F F F 21 420-439 32 32 32 52 52 52 F F F 22 440-459 32 32 32 52 52 52 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 32 32 32 52 52 52 F F F 25 500-519 28 28 28 45 45 45 F F F 26 520-539 32 32 32 53 53 53 F F F 27 540-559 32 32 32 54 54 54 T T T 28 560-579 32 32 32 52 52 52 T T T 29 580-599 24 24 24 40 40 40 F F F Max gen: 32, Max flush: 54 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 30 30 30 47 47 47 F F F 2 040-059 30 30 30 48 48 48 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 31 31 31 50 50 50 F F F 6 120-139 28 28 28 45 45 45 F F F 7 140-159 33 33 33 55 55 55 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 33 33 33 55 55 55 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 32 32 32 52 52 52 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 28 28 28 45 45 45 F F F 19 380-399 32 32 32 52 52 52 F F F 20 400-419 32 32 32 52 52 52 F F F 21 420-439 32 32 32 52 52 52 F F F 22 440-459 32 32 32 52 52 52 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 32 32 32 52 52 52 F F F 25 500-519 28 28 28 45 45 45 F F F 26 520-539 32 32 32 53 53 53 F F F 27 540-559 33 33 33 55 55 55 F F F 28 560-579 33 33 33 55 55 55 F F F 29 580-599 24 24 24 40 40 40 F F F Max gen: 33, Max flush: 55 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.059614905Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.104490751Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.105249458Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.105289542Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.105304813Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.105318935Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326} {"msg":"Crucible c92ce1a3-4437-4b60-a437-0d65f7651040 has session id: c081b2bf-b9e9-473f-92ac-02f9a331a5ee","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.105889332Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Upstairs opts: Upstairs UUID: c92ce1a3-4437-4b60-a437-0d65f7651040, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.105932787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Crucible stats registered with UUID: c92ce1a3-4437-4b60-a437-0d65f7651040","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.105957329Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:47.105972401Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106407565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106430447Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326} {"msg":"c92ce1a3-4437-4b60-a437-0d65f7651040 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106500864Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106517696Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"0","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106539388Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"0","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106552659Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"1","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106566301Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"1","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106578942Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106592533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106605094Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106619686Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106639098Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:47.106769711Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:64952)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106807535Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:47.106836468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:47.106865971Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:47726)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106889503Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.106914586Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.107486033Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"0","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.107525897Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"1","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.108156991Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"1","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.108267692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"0","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.108791065Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:47.108887295Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.108304806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:46281)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.108472683Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.109451872Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.11002944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[0]R flush_numbers[0..12]: [45, 47, 48, 46, 28, 50, 45, 55, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110608218Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[0]R generation[0..12]: [28, 30, 30, 30, 15, 31, 28, 33, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.11062674Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110642782Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[1]R flush_numbers[0..12]: [45, 47, 48, 46, 28, 50, 45, 55, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110657023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[1]R generation[0..12]: [28, 30, 30, 30, 15, 31, 28, 33, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110670534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110683956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[2]R flush_numbers[0..12]: [45, 47, 48, 46, 28, 50, 45, 55, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110697357Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[2]R generation[0..12]: [28, 30, 30, 30, 15, 31, 28, 33, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110710549Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.11072417Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110736601Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110749233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110761584Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110784186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"":"downstairs","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110803338Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110815539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"0","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110828881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"1","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110841492Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"client":"2","":"downstairs","session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110854313Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110866114Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"c92ce1a3-4437-4b60-a437-0d65f7651040 is now active with session: c081b2bf-b9e9-473f-92ac-02f9a331a5ee","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110878506Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110890047Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326,"session_id":"c081b2bf-b9e9-473f-92ac-02f9a331a5ee"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:25:57.110901848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 569 len 20480 data: 3 2 2 3 3 02/30 Read block 171 len 20480 03/30 Read block 156 len 32768 04/30 Write block 39 len 36864 data: 5 5 4 4 5 3 4 4 4 05/30 Read block 34 len 28672 06/30 Write block 3 len 20480 data: 5 5 4 4 4 07/30 Flush 08/30 Write block 126 len 20480 data: 5 4 4 3 3 09/30 Read block 547 len 36864 10/30 Write block 379 len 8192 data: 3 2 11/30 Write block 520 len 12288 data: 4 3 4 12/30 Read block 449 len 20480 13/30 Read block 87 len 8192 14/30 Read block 174 len 8192 15/30 Read block 170 len 24576 16/30 Write block 116 len 40960 data: 6 8 8 8 8 7 5 5 5 5 17/30 Write block 423 len 8192 data: 3 3 18/30 Write block 248 len 32768 data: 4 5 6 6 7 7 6 4 19/30 Read block 472 len 32768 20/30 Read block 435 len 32768 21/30 Read block 175 len 20480 22/30 Read block 271 len 20480 23/30 Write block 514 len 36864 data: 4 4 4 4 5 5 5 4 5 24/30 Read block 82 len 20480 25/30 Read block 39 len 36864 26/30 Write block 574 len 36864 data: 3 4 4 4 4 4 4 4 4 27/30 Write block 135 len 40960 data: 6 7 6 5 6 6 6 6 5 3 28/30 Write block 32 len 32768 data: 3 3 3 4 4 5 5 6 29/30 Write block 556 len 24576 data: 3 4 4 5 3 3 30/30 Write block 545 len 36864 data: 5 5 5 5 5 5 4 2 2 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:25:57.132204016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 56 56 56 F F F 1 020-039 34 34 34 56 56 56 T T T 2 040-059 34 34 34 56 56 56 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 31 31 31 50 50 50 T T T 6 120-139 28 28 28 45 45 45 T T T 7 140-159 33 33 33 55 55 55 T T T 8 160-179 20 20 20 34 34 34 F F F 9 180-199 33 33 33 55 55 55 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 32 32 32 52 52 52 T T T 13 260-279 23 23 23 38 38 38 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 28 28 28 45 45 45 T T T 19 380-399 32 32 32 52 52 52 T T T 20 400-419 32 32 32 52 52 52 F F F 21 420-439 32 32 32 52 52 52 T T T 22 440-459 32 32 32 52 52 52 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 32 32 32 52 52 52 F F F 25 500-519 28 28 28 45 45 45 T T T 26 520-539 32 32 32 53 53 53 T T T 27 540-559 33 33 33 55 55 55 T T T 28 560-579 34 34 34 56 56 56 T T T 29 580-599 24 24 24 40 40 40 T T T Max gen: 34, Max flush: 56 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 56 56 56 F F F 1 020-039 35 35 35 57 57 57 F F F 2 040-059 34 34 34 56 56 56 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 35 35 35 57 57 57 F F F 6 120-139 35 35 35 57 57 57 F F F 7 140-159 35 35 35 57 57 57 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 33 33 33 55 55 55 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 35 35 35 57 57 57 F F F 13 260-279 23 23 23 38 38 38 F F F 14 280-299 30 30 30 47 47 47 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 35 35 35 57 57 57 F F F 19 380-399 35 35 35 57 57 57 F F F 20 400-419 32 32 32 52 52 52 F F F 21 420-439 35 35 35 57 57 57 F F F 22 440-459 32 32 32 52 52 52 F F F 23 460-479 30 30 30 46 46 46 F F F 24 480-499 32 32 32 52 52 52 F F F 25 500-519 35 35 35 57 57 57 F F F 26 520-539 35 35 35 57 57 57 F F F 27 540-559 35 35 35 57 57 57 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 35, Max flush: 57 Begin loop 18 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.417198978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.462656003Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.46341966Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.463471765Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.463489467Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.463506688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334} {"msg":"Crucible 104d1217-9c0b-46f8-9306-c43cdd4deb11 has session id: ecb8c819-6315-4121-b4fa-96d4f871e72c","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464069235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Upstairs opts: Upstairs UUID: 104d1217-9c0b-46f8-9306-c43cdd4deb11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464107859Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Crucible stats registered with UUID: 104d1217-9c0b-46f8-9306-c43cdd4deb11","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464135662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:11.464153664Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464648904Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464678707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334} {"msg":"104d1217-9c0b-46f8-9306-c43cdd4deb11 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464753884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464771596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"0","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464795648Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"0","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.4648091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"1","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464822881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"1","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464835752Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"2","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464849694Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"2","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464862895Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464876256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.464897669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48249)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.465072006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"ds_connection connected from Ok(127.0.0.1:50635)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.465124721Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63463)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.465155355Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.465852305Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"2","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.465890089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"0","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.466481598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"0","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.466632884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"1","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.466661056Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"2","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.467122353Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.467296331Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.467375119Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"1","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 56, 46, 28, 57, 57, 57, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.467902382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"[0]R generation[0..12]: [34, 35, 34, 30, 15, 35, 35, 35, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.467929844Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.467947746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 56, 46, 28, 57, 57, 57, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.467964748Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"[1]R generation[0..12]: [34, 35, 34, 30, 15, 35, 35, 35, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.46798724Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468005142Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 56, 46, 28, 57, 57, 57, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468021644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [34, 35, 34, 30, 15, 35, 35, 35, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468044406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468061578Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468077259Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468093191Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468108653Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468124424Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"":"downstairs","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468140556Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468154977Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"0","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468171139Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"1","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.46818723Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"client":"2","":"downstairs","session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468203292Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468218144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 104d1217-9c0b-46f8-9306-c43cdd4deb11 is now active with session: ecb8c819-6315-4121-b4fa-96d4f871e72c","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468238355Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468253787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334,"session_id":"ecb8c819-6315-4121-b4fa-96d4f871e72c"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:11.468268659Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 527 len 16384 data: 4 4 6 6 02/30 Flush 03/30 Write block 488 len 28672 data: 4 4 3 3 3 3 3 04/30 Read block 497 len 12288 05/30 Flush 06/30 Write block 53 len 8192 data: 3 5 07/30 Flush 08/30 Write block 108 len 12288 data: 5 5 6 09/30 Read block 28 len 20480 10/30 Read block 564 len 36864 11/30 Write block 400 len 12288 data: 2 2 2 12/30 Write block 467 len 40960 data: 7 7 7 7 6 4 3 2 2 2 13/30 Write block 134 len 28672 data: 7 7 8 7 6 7 7 14/30 Read block 372 len 4096 15/30 Read block 530 len 32768 16/30 Read block 314 len 4096 17/30 Read block 311 len 24576 18/30 Read block 521 len 28672 19/30 Write block 103 len 4096 data: 3 20/30 Write block 395 len 32768 data: 4 4 3 2 2 3 3 3 21/30 Read block 172 len 8192 22/30 Write block 276 len 8192 data: 3 3 23/30 Flush 24/30 Write block 417 len 4096 data: 2 25/30 Write block 280 len 32768 data: 2 2 2 2 2 2 3 4 26/30 Write block 391 len 28672 data: 6 6 6 5 5 5 4 27/30 Read block 424 len 24576 28/30 Write block 438 len 20480 data: 6 7 5 4 4 29/30 Write block 132 len 40960 data: 5 6 8 8 9 8 7 8 8 7 30/30 Read block 103 len 4096 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:11.487579476Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 56 56 56 F F F 1 020-039 35 35 35 57 57 57 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 36 36 36 61 61 61 F F F 6 120-139 36 36 36 61 61 61 T T T 7 140-159 36 36 36 61 61 61 T T T 8 160-179 20 20 20 34 34 34 F F F 9 180-199 33 33 33 55 55 55 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 35 35 35 57 57 57 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 30 30 30 47 47 47 T T T 15 300-319 30 30 30 47 47 47 F F F 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 35 35 35 57 57 57 F F F 19 380-399 36 36 36 61 61 61 T T T 20 400-419 36 36 36 61 61 61 T T T 21 420-439 35 35 35 57 57 57 T T T 22 440-459 32 32 32 52 52 52 T T T 23 460-479 36 36 36 61 61 61 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 35 35 35 57 57 57 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 35 35 35 57 57 57 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 36, Max flush: 61 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 56 56 56 F F F 1 020-039 35 35 35 57 57 57 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 36 36 36 61 61 61 F F F 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 33 33 33 55 55 55 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 35 35 35 57 57 57 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 37 37 37 62 62 62 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 35 35 35 57 57 57 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 37 37 37 62 62 62 F F F 21 420-439 37 37 37 62 62 62 F F F 22 440-459 37 37 37 62 62 62 F F F 23 460-479 36 36 36 61 61 61 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 35 35 35 57 57 57 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 35 35 35 57 57 57 F F F 28 560-579 35 35 35 57 57 57 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 37, Max flush: 62 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.743018766Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.787246777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.788080881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.788125015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.788144997Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.788159449Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342} {"msg":"Crucible 39cb77e3-49a6-47c8-9957-4a3d76e7d19d has session id: 10daad36-bce5-41b8-a9d0-e7a3b1e9839f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.788816465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Upstairs opts: Upstairs UUID: 39cb77e3-49a6-47c8-9957-4a3d76e7d19d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.788847078Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Crucible stats registered with UUID: 39cb77e3-49a6-47c8-9957-4a3d76e7d19d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.78886171Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:25.788874381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789394813Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789419606Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342} {"msg":"39cb77e3-49a6-47c8-9957-4a3d76e7d19d active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789476882Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789503174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"0","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789528737Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"0","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789543648Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.78955768Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789570941Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"2","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789584742Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"2","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789651349Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789714015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789743498Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40218)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789767751Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37400)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.789918376Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:25.78995507Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:25.789986333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:25.790013046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.790078122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.790435318Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"2","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.790462651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"0","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.791080233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"0","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.791225288Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"2","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.791661902Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:25.7917461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.790926408Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38128)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.791101576Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.792203537Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.792773484Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[0]R flush_numbers[0..12]: [56, 57, 60, 46, 28, 61, 62, 62, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793403218Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[0]R generation[0..12]: [34, 35, 36, 30, 15, 36, 37, 37, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.79342831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793447762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[1]R flush_numbers[0..12]: [56, 57, 60, 46, 28, 61, 62, 62, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793465334Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[1]R generation[0..12]: [34, 35, 36, 30, 15, 36, 37, 37, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793482216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793499237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[2]R flush_numbers[0..12]: [56, 57, 60, 46, 28, 61, 62, 62, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793515939Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[2]R generation[0..12]: [34, 35, 36, 30, 15, 36, 37, 37, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793532761Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793549712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:26:35.793571565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793590176Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793606248Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793622Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"":"downstairs","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793648332Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793664304Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"0","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793680596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"1","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793696447Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"client":"2","":"downstairs","session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793712669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.79372758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"39cb77e3-49a6-47c8-9957-4a3d76e7d19d is now active with session: 10daad36-bce5-41b8-a9d0-e7a3b1e9839f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793743332Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793757823Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342,"session_id":"10daad36-bce5-41b8-a9d0-e7a3b1e9839f"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:35.793772585Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 12288 data: 7 8 8 02/30 Read block 143 len 28672 03/30 Read block 530 len 40960 04/30 Write block 112 len 40960 data: 5 4 4 5 7 9 9 9 9 8 05/30 Read block 415 len 28672 06/30 Read block 189 len 20480 07/30 Read block 475 len 28672 08/30 Flush 09/30 Write block 395 len 12288 data: 6 6 5 10/30 Read block 220 len 12288 11/30 Read block 269 len 16384 12/30 Write block 410 len 36864 data: 6 3 5 5 4 4 3 3 3 13/30 Write block 573 len 12288 data: 4 4 5 14/30 Write block 367 len 8192 data: 2 2 15/30 Read block 599 len 4096 16/30 Read block 577 len 16384 17/30 Read block 430 len 4096 18/30 Read block 114 len 8192 19/30 Flush 20/30 Write block 300 len 20480 data: 6 5 6 6 8 21/30 Write block 246 len 36864 data: 3 3 5 6 7 8 9 9 7 22/30 Write block 130 len 8192 data: 4 4 23/30 Read block 462 len 8192 24/30 Write block 22 len 24576 data: 5 5 4 3 3 4 25/30 Read block 72 len 8192 26/30 Read block 517 len 20480 27/30 Read block 574 len 8192 28/30 Read block 367 len 24576 29/30 Write block 366 len 8192 data: 2 3 30/30 Read block 189 len 4096 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 11 11 9 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:35.81384796Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 56 56 56 F F F 1 020-039 35 35 35 57 57 57 T T T 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 38 38 38 63 63 63 T T T 7 140-159 37 37 37 62 62 62 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 33 33 33 55 55 55 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 38 38 38 63 63 63 T T T 13 260-279 36 36 36 61 61 61 F F F 14 280-299 37 37 37 62 62 62 F F F 15 300-319 30 30 30 47 47 47 T T T 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 38 38 38 64 64 64 T T T 19 380-399 38 38 38 64 64 64 F F F 20 400-419 38 38 38 64 64 64 F F F 21 420-439 37 37 37 62 62 62 F F F 22 440-459 37 37 37 62 62 62 F F F 23 460-479 36 36 36 61 61 61 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 35 35 35 57 57 57 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 35 35 35 57 57 57 F F F 28 560-579 38 38 38 64 64 64 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 38, Max flush: 64 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 56 56 56 F F F 1 020-039 39 39 39 65 65 65 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 39 39 39 65 65 65 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 33 33 33 55 55 55 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 31 31 31 50 50 50 F F F 12 240-259 39 39 39 65 65 65 F F F 13 260-279 36 36 36 61 61 61 F F F 14 280-299 37 37 37 62 62 62 F F F 15 300-319 39 39 39 65 65 65 F F F 16 320-339 31 31 31 50 50 50 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 38 38 38 64 64 64 F F F 20 400-419 38 38 38 64 64 64 F F F 21 420-439 37 37 37 62 62 62 F F F 22 440-459 37 37 37 62 62 62 F F F 23 460-479 36 36 36 61 61 61 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 35 35 35 57 57 57 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 35 35 35 57 57 57 F F F 28 560-579 38 38 38 64 64 64 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 39, Max flush: 65 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.058100882Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.102924233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.10378686Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.103825484Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.103840335Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.103854917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350} {"msg":"Crucible 73af897f-5800-4dcc-bdbd-7510e615380b has session id: 73d532fb-1c70-4d13-9dd2-c46790efabfa","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.104459788Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Upstairs opts: Upstairs UUID: 73af897f-5800-4dcc-bdbd-7510e615380b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.104501562Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Crucible stats registered with UUID: 73af897f-5800-4dcc-bdbd-7510e615380b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.104531045Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:40.104553557Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.104957728Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.104985641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350} {"msg":"73af897f-5800-4dcc-bdbd-7510e615380b active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105104903Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105130236Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"0","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105155538Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"0","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.10517637Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"1","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105211174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"1","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105237926Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105265359Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105289761Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105313034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105335016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:42220)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105368309Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:61467)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105391112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:40.105425955Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:40.105463539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:40.10567318Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.105746858Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.106159009Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"1","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.106187282Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"0","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.106872281Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"1","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.106912625Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"0","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.107573962Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:40.107602635Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.10735101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:58420)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.107539869Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.108420667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109017378Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[0]R flush_numbers[0..12]: [56, 65, 60, 46, 28, 63, 65, 62, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.10963667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[0]R generation[0..12]: [34, 39, 36, 30, 15, 38, 39, 37, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109661523Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109677314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[1]R flush_numbers[0..12]: [56, 65, 60, 46, 28, 63, 65, 62, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109691406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[1]R generation[0..12]: [34, 39, 36, 30, 15, 38, 39, 37, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109704967Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109718838Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[2]R flush_numbers[0..12]: [56, 65, 60, 46, 28, 63, 65, 62, 34, 55, 51, 50]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.10973276Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[2]R generation[0..12]: [34, 39, 36, 30, 15, 38, 39, 37, 20, 33, 32, 31]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109746151Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109759793Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109772444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109785195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109797777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9EWait for a query_work_queue command to finish before sending IO ","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109817038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"":"downstairs","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.10982983Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109844721Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"0","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109858183Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"1","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109871194Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"client":"2","":"downstairs","session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109884365Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109896366Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"73af897f-5800-4dcc-bdbd-7510e615380b is now active with session: 73d532fb-1c70-4d13-9dd2-c46790efabfa","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109908838Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.109920479Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350,"session_id":"73d532fb-1c70-4d13-9dd2-c46790efabfa"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:50.10993247Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 139 len 20480 data: 9 9 8 7 6 02/30 Read block 170 len 36864 03/30 Read block 292 len 4096 04/30 Read block 111 len 32768 05/30 Flush 06/30 Write block 545 len 36864 data: 6 6 6 6 6 6 5 3 3 07/30 Write block 458 len 8192 data: 4 4 08/30 Read block 437 len 28672 09/30 Read block 540 len 4096 10/30 Read block 499 len 32768 11/30 Read block 167 len 32768 12/30 Write block 485 len 24576 data: 3 3 3 5 5 4 13/30 Write block 323 len 20480 data: 5 4 4 4 3 14/30 Flush 15/30 Write block 519 len 4096 data: 6 16/30 Write block 550 len 4096 data: 7 17/30 Read block 62 len 20480 18/30 Read block 298 len 4096 19/30 Read block 286 len 24576 20/30 Read block 99 len 32768 21/30 Read block 364 len 28672 22/30 Write block 222 len 40960 data: 5 5 5 6 6 4 4 3 4 5 23/30 Write block 259 len 24576 data: 4 3 3 2 3 3 24/30 Read block 82 len 28672 25/30 Read block 441 len 32768 26/30 Flush 27/30 Write block 191 len 28672 data: 2 2 3 3 4 2 3 28/30 Read block 325 len 8192 29/30 Write block 439 len 12288 data: 8 6 5 30/30 Read block 335 len 12288 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 3 4 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:50.130401044Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 56 56 56 F F F 1 020-039 39 39 39 65 65 65 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 40 40 40 66 66 66 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 33 33 33 55 55 55 T T T 10 200-219 32 32 32 51 51 51 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 40 40 40 68 68 68 F F F 13 260-279 40 40 40 68 68 68 F F F 14 280-299 37 37 37 62 62 62 F F F 15 300-319 39 39 39 65 65 65 F F F 16 320-339 40 40 40 67 67 67 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 38 38 38 64 64 64 F F F 20 400-419 38 38 38 64 64 64 F F F 21 420-439 37 37 37 62 62 62 T T T 22 440-459 40 40 40 67 67 67 T T T 23 460-479 36 36 36 61 61 61 F F F 24 480-499 40 40 40 67 67 67 F F F 25 500-519 40 40 40 68 68 68 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 68 68 68 F F F 28 560-579 38 38 38 64 64 64 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 40, Max flush: 68 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 56 56 56 F F F 1 020-039 39 39 39 65 65 65 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 40 40 40 66 66 66 F F F 8 160-179 20 20 20 34 34 34 F F F 9 180-199 41 41 41 69 69 69 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 40 40 40 68 68 68 F F F 13 260-279 40 40 40 68 68 68 F F F 14 280-299 37 37 37 62 62 62 F F F 15 300-319 39 39 39 65 65 65 F F F 16 320-339 40 40 40 67 67 67 F F F 17 340-359 24 24 24 40 40 40 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 38 38 38 64 64 64 F F F 20 400-419 38 38 38 64 64 64 F F F 21 420-439 41 41 41 69 69 69 F F F 22 440-459 41 41 41 69 69 69 F F F 23 460-479 36 36 36 61 61 61 F F F 24 480-499 40 40 40 67 67 67 F F F 25 500-519 40 40 40 68 68 68 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 68 68 68 F F F 28 560-579 38 38 38 64 64 64 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 41, Max flush: 69 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.358245342Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.402914807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.403794996Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.403846611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.403864403Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.403881925Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358} {"msg":"Crucible 02d43715-7d26-4182-b428-ab00d960bc75 has session id: c6e431de-a08d-4267-a388-21143e093f41","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.404440911Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Upstairs opts: Upstairs UUID: 02d43715-7d26-4182-b428-ab00d960bc75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.404475534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Crucible stats registered with UUID: 02d43715-7d26-4182-b428-ab00d960bc75","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.404491106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:54.404504197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.404954713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.404978045Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358} {"msg":"02d43715-7d26-4182-b428-ab00d960bc75 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405059864Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405077415Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"0","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405100338Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"0","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405117919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"1","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405132161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"1","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405145052Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"2","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405167034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"2","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405193667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.40521948Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405243642Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"ds_connection connected from Ok(127.0.0.1:39667)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405357023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"ds_connection connected from Ok(127.0.0.1:51628)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.405392387Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"ds_connection connected from Ok(127.0.0.1:51900)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.40541616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.406074326Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"2","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.406101389Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"0","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.40680691Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"0","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.406844633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"2","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.407478877Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.40750461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.407978748Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"1","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.40849891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"1","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[0]R flush_numbers[0..12]: [56, 65, 60, 46, 28, 63, 66, 66, 34, 69, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.40908617Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[0]R generation[0..12]: [34, 39, 36, 30, 15, 38, 40, 40, 20, 41, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409106662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409121223Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[1]R flush_numbers[0..12]: [56, 65, 60, 46, 28, 63, 66, 66, 34, 69, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409134504Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[1]R generation[0..12]: [34, 39, 36, 30, 15, 38, 40, 40, 20, 41, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409147656Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409161687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[2]R flush_numbers[0..12]: [56, 65, 60, 46, 28, 63, 66, 66, 34, 69, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409175019Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[2]R generation[0..12]: [34, 39, 36, 30, 15, 38, 40, 40, 20, 41, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.40918848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409202611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409215423Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409233545Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:26:54.409246716Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409264588Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"":"downstairs","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409277499Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.40928921Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"0","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409302051Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"1","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409314783Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"client":"2","":"downstairs","session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409327724Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409339585Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"02d43715-7d26-4182-b428-ab00d960bc75 is now active with session: c6e431de-a08d-4267-a388-21143e093f41","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409352097Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409363818Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358,"session_id":"c6e431de-a08d-4267-a388-21143e093f41"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:54.409375729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 161 len 16384 data: 2 2 3 3 02/30 Write block 19 len 16384 data: 7 6 5 6 03/30 Read block 166 len 24576 04/30 Read block 476 len 4096 05/30 Read block 583 len 12288 06/30 Write block 30 len 12288 data: 3 3 4 07/30 Read block 23 len 4096 08/30 Write block 498 len 40960 data: 3 4 3 3 2 2 2 3 4 3 09/30 Read block 22 len 20480 10/30 Write block 31 len 12288 data: 4 5 4 11/30 Read block 244 len 20480 12/30 Write block 342 len 8192 data: 5 5 13/30 Write block 391 len 32768 data: 7 7 7 6 7 7 6 3 14/30 Read block 447 len 8192 15/30 Write block 244 len 16384 data: 3 3 4 4 16/30 Read block 359 len 20480 17/30 Read block 422 len 20480 18/30 Read block 388 len 24576 19/30 Read block 6 len 16384 20/30 Write block 330 len 36864 data: 6 5 7 7 8 8 8 5 6 21/30 Read block 446 len 12288 22/30 Read block 322 len 36864 23/30 Read block 267 len 4096 24/30 Read block 37 len 24576 25/30 Write block 261 len 8192 data: 4 3 26/30 Flush 27/30 Write block 578 len 8192 data: 5 5 28/30 Read block 511 len 24576 29/30 Read block 166 len 32768 30/30 Read block 84 len 28672 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:54.429360555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 70 70 70 F F F 1 020-039 42 42 42 70 70 70 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 40 40 40 66 66 66 F F F 8 160-179 42 42 42 70 70 70 F F F 9 180-199 41 41 41 69 69 69 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 42 42 42 70 70 70 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 37 37 37 62 62 62 F F F 15 300-319 39 39 39 65 65 65 F F F 16 320-339 42 42 42 70 70 70 F F F 17 340-359 42 42 42 70 70 70 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 38 38 38 64 64 64 F F F 21 420-439 41 41 41 69 69 69 F F F 22 440-459 41 41 41 69 69 69 F F F 23 460-479 36 36 36 61 61 61 F F F 24 480-499 42 42 42 70 70 70 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 68 68 68 F F F 28 560-579 38 38 38 64 64 64 T T T 29 580-599 35 35 35 57 57 57 F F F Max gen: 42, Max flush: 70 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 70 70 70 F F F 1 020-039 42 42 42 70 70 70 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 40 40 40 66 66 66 F F F 8 160-179 42 42 42 70 70 70 F F F 9 180-199 41 41 41 69 69 69 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 42 42 42 70 70 70 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 37 37 37 62 62 62 F F F 15 300-319 39 39 39 65 65 65 F F F 16 320-339 42 42 42 70 70 70 F F F 17 340-359 42 42 42 70 70 70 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 38 38 38 64 64 64 F F F 21 420-439 41 41 41 69 69 69 F F F 22 440-459 41 41 41 69 69 69 F F F 23 460-479 36 36 36 61 61 61 F F F 24 480-499 42 42 42 70 70 70 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 40 40 40 68 68 68 F F F 28 560-579 43 43 43 71 71 71 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 43, Max flush: 71 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.652936271Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:58.696284653Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.69675279Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.728456038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.729128836Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.72917023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.729187351Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.729204513Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366} {"msg":"Crucible cdcd7347-b5a9-4e8d-877d-b2a0efe5e599 has session id: a1d28c29-27c5-43f4-ba37-24dc8cd6ac39","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.729794463Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Upstairs opts: Upstairs UUID: cdcd7347-b5a9-4e8d-877d-b2a0efe5e599, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.729825116Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Crucible stats registered with UUID: cdcd7347-b5a9-4e8d-877d-b2a0efe5e599","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.729840717Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:58.729858499Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730287242Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730310185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366} {"msg":"cdcd7347-b5a9-4e8d-877d-b2a0efe5e599 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730377101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730393583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"0","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730416755Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"0","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730430427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"1","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730444388Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"1","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730457239Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"2","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730470821Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"2","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730491273Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730513865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730537878Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"ds_connection connected from Ok(127.0.0.1:48666)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730599564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"ds_connection connected from Ok(127.0.0.1:53643)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.730642318Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"ds_connection connected from Ok(127.0.0.1:43818)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.73085652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.731412476Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"2","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.73144979Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"1","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.731688484Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"0","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.732154921Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"1","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.732233189Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"2","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.73254447Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"0","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.73284341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.732912617Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[0]R flush_numbers[0..12]: [70, 70, 60, 46, 28, 63, 66, 66, 70, 69, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733161492Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[0]R generation[0..12]: [42, 42, 36, 30, 15, 38, 40, 40, 42, 41, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733183074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733197846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[1]R flush_numbers[0..12]: [70, 70, 60, 46, 28, 63, 66, 66, 70, 69, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733212347Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[1]R generation[0..12]: [42, 42, 36, 30, 15, 38, 40, 40, 42, 41, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733225739Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.73323934Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[2]R flush_numbers[0..12]: [70, 70, 60, 46, 28, 63, 66, 66, 70, 69, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733256892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[2]R generation[0..12]: [42, 42, 36, 30, 15, 38, 40, 40, 42, 41, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733270693Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733284404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733296886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733309717Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Next flush: 72","v":0,"nameWait for a query_work_queue command to finish before sending IO ":"crucible","level":30,"time":"2025-02-27T17:26:58.73334185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733367043Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"":"downstairs","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733390015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733403316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"0","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733416688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"1","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733429689Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"client":"2","":"downstairs","session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733449661Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733471654Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"cdcd7347-b5a9-4e8d-877d-b2a0efe5e599 is now active with session: a1d28c29-27c5-43f4-ba37-24dc8cd6ac39","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733495976Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.733517118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366,"session_id":"a1d28c29-27c5-43f4-ba37-24dc8cd6ac39"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:26:58.73353307Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 319 len 40960 data: 4 4 4 5 6 5 5 5 4 4 02/30 Read block 430 len 4096 03/30 Read block 64 len 20480 04/30 Read block 339 len 16384 05/30 Read block 29 len 8192 06/30 Read block 461 len 24576 07/30 Read block 520 len 24576 08/30 Read block 512 len 32768 09/30 Write block 249 len 28672 data: 7 8 9 10 10 8 5 10/30 Flush 11/30 Flush 12/30 Write block 552 len 16384 data: 4 4 2 2 13/30 Read block 549 len 28672 14/30 Write block 241 len 24576 data: 2 2 3 4 4 5 15/30 Flush 16/30 Write block 184 len 4096 data: 5 17/30 Write block 155 len 20480 data: 3 3 3 3 2 18/30 Write block 356 len 8192 data: 2 2 19/30 Read block 374 len 4096 20/30 Flush 21/30 Write block 336 len 12288 data: 9 6 7 22/30 Read block 439 len 36864 23/30 Read block 236 len 8192 24/30 Write block 408 len 24576 data: 7 7 7 4 6 6 25/30 Read block 255 len 32768 26/30 Flush 27/30 Write block 543 len 4096 data: 6 28/30 Write block 185 len 8192 data: 5 4 29/30 Write block 13 len 12288 data: 3 3 6 30/30 Write block 299 len 20480 data: 5 7 6 7 7 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:26:58.752281671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 70 70 70 T T T 1 020-039 42 42 42 70 70 70 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 44 44 44 75 75 75 F F F 8 160-179 42 42 42 70 70 70 F F F 9 180-199 44 44 44 75 75 75 T T T 10 200-219 32 32 32 51 51 51 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 37 37 37 62 62 62 T T T 15 300-319 44 44 44 72 72 72 T T T 16 320-339 44 44 44 76 76 76 F F F 17 340-359 44 44 44 75 75 75 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 44 44 44 76 76 76 F F F 21 420-439 41 41 41 69 69 69 F F F 22 440-459 41 41 41 69 69 69 F F F 23 460-479 36 36 36 61 61 61 F F F 24 480-499 42 42 42 70 70 70 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 44 44 44 74 74 74 T T T 28 560-579 43 43 43 71 71 71 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 44, Max flush: 76 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 77 77 77 F F F 1 020-039 42 42 42 70 70 70 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 30 30 30 46 46 46 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 38 38 38 63 63 63 F F F 6 120-139 40 40 40 66 66 66 F F F 7 140-159 44 44 44 75 75 75 F F F 8 160-179 42 42 42 70 70 70 F F F 9 180-199 45 45 45 77 77 77 F F F 10 200-219 32 32 32 51 51 51 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 45 45 45 77 77 77 F F F 15 300-319 45 45 45 77 77 77 F F F 16 320-339 44 44 44 76 76 76 F F F 17 340-359 44 44 44 75 75 75 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 44 44 44 76 76 76 F F F 21 420-439 41 41 41 69 69 69 F F F 22 440-459 41 41 41 69 69 69 F F F 23 460-479 36 36 36 61 61 61 F F F 24 480-499 42 42 42 70 70 70 F F F 25 500-519 42 42 42 70 70 70 F F F 26 520-539 36 36 36 58 58 58 F F F 27 540-559 45 45 45 77 77 77 F F F 28 560-579 43 43 43 71 71 71 F F F 29 580-599 35 35 35 57 57 57 F F F Max gen: 45, Max flush: 77 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.026774783Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.074453262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.075439291Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.075487416Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.075503458Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.075518929Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374} {"msg":"Crucible 71bfd540-9162-4c8b-8b7f-100acdfc1de9 has session id: 512b54f1-1f26-4652-9299-69c972340521","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076034341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Upstairs opts: Upstairs UUID: 71bfd540-9162-4c8b-8b7f-100acdfc1de9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076114819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Crucible stats registered with UUID: 71bfd540-9162-4c8b-8b7f-100acdfc1de9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076135261Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:03.076147952Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076642913Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076678776Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374} {"msg":"71bfd540-9162-4c8b-8b7f-100acdfc1de9 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076730101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076755154Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"0","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076778636Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"0","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076796818Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"1","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076823481Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"1","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076847733Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"2","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076874126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"2","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076898358Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076922791Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076949373Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"ds_connection connected from Ok(127.0.0.1:43024)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.076977206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"ds_connection connected from Ok(127.0.0.1:49082)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.077031662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"ds_connection connected from Ok(127.0.0.1:64655)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.077059784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.07771187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"0","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.077749784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"2","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.078360016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"0","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.07850262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"2","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.0786001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"1","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.078907961Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079122683Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079210501Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"1","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[0]R flush_numbers[0..12]: [77, 70, 60, 46, 28, 63, 66, 75, 70, 77, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079742725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[0]R generation[0..12]: [45, 42, 36, 30, 15, 38, 40, 44, 42, 45, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079765177Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079780179Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[1]R flush_numbers[0..12]: [77, 70, 60, 46, 28, 63, 66, 75, 70, 77, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079798681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[1]R generation[0..12]: [45, 42, 36, 30, 15, 38, 40, 44, 42, 45, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079812502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079826234Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[2]R flush_numbers[0..12]: [77, 70, 60, 46, 28, 63, 66, 75, 70, 77, 51, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079839685Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[2]R generation[0..12]: [45, 42, 36, 30, 15, 38, 40, 44, 42, 45, 32, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079852846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:27:03.079866447Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.07989248Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079912782Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079928144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079949226Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"":"downstairs","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079973858Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.079997991Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"0","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.080018393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"1","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.080032384Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"client":"2","":"downstairs","session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.080049356Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.080078969Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"71bfd540-9162-4c8b-8b7f-100acdfc1de9 is now active with session: 512b54f1-1f26-4652-9299-69c972340521","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.080101571Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.080120633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374,"session_id":"512b54f1-1f26-4652-9299-69c972340521"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:03.080141095Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 151 len 4096 data: 4 02/30 Read block 392 len 16384 03/30 Read block 251 len 36864 04/30 Write block 598 len 8192 data: 2 2 05/30 Write block 452 len 32768 data: 4 4 4 4 4 4 5 5 06/30 Write block 412 len 32768 data: 7 7 5 5 4 4 4 3 07/30 Read block 61 len 16384 08/30 Write block 120 len 36864 data: 10 9 6 6 6 6 6 5 5 09/30 Read block 40 len 8192 10/30 Write block 547 len 20480 data: 7 7 7 8 6 11/30 Read block 113 len 16384 12/30 Write block 106 len 36864 data: 5 5 6 6 7 5 6 5 5 13/30 Write block 149 len 16384 data: 5 4 5 4 14/30 Read block 592 len 32768 15/30 Read block 262 len 12288 16/30 Write block 514 len 40960 data: 5 5 5 5 6 7 6 5 6 4 17/30 Write block 591 len 16384 data: 4 3 3 3 18/30 Read block 242 len 4096 19/30 Write block 68 len 8192 data: 4 3 20/30 Read block 570 len 4096 21/30 Read block 473 len 8192 22/30 Flush 23/30 Write block 196 len 32768 data: 3 4 3 5 4 5 6 4 24/30 Read block 427 len 32768 25/30 Read block 72 len 20480 26/30 Read block 444 len 24576 27/30 Write block 157 len 36864 data: 4 4 3 2 3 3 4 4 4 28/30 Write block 471 len 12288 data: 7 5 4 29/30 Write block 418 len 20480 data: 5 4 2 3 3 30/30 Write block 174 len 12288 data: 4 4 4 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 9 Sent Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:03.100008329Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 77 77 77 F F F 1 020-039 42 42 42 70 70 70 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 46 46 46 78 78 78 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 46 46 46 78 78 78 F F F 6 120-139 46 46 46 78 78 78 F F F 7 140-159 46 46 46 78 78 78 T T T 8 160-179 42 42 42 70 70 70 T T T 9 180-199 45 45 45 77 77 77 T T T 10 200-219 32 32 32 51 51 51 T T T 11 220-239 40 40 40 68 68 68 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 45 45 45 77 77 77 F F F 15 300-319 45 45 45 77 77 77 F F F 16 320-339 44 44 44 76 76 76 F F F 17 340-359 44 44 44 75 75 75 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 46 46 46 78 78 78 T T T 21 420-439 41 41 41 69 69 69 T T T 22 440-459 46 46 46 78 78 78 F F F 23 460-479 36 36 36 61 61 61 T T T 24 480-499 42 42 42 70 70 70 F F F 25 500-519 46 46 46 78 78 78 F F F 26 520-539 46 46 46 78 78 78 F F F 27 540-559 46 46 46 78 78 78 F F F 28 560-579 43 43 43 71 71 71 F F F 29 580-599 46 46 46 78 78 78 F F F Max gen: 46, Max flush: 78 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 77 77 77 F F F 1 020-039 42 42 42 70 70 70 F F F 2 040-059 36 36 36 60 60 60 F F F 3 060-079 46 46 46 78 78 78 F F F 4 080-099 15 15 15 28 28 28 F F F 5 100-119 46 46 46 78 78 78 F F F 6 120-139 46 46 46 78 78 78 F F F 7 140-159 47 47 47 79 79 79 F F F 8 160-179 47 47 47 79 79 79 F F F 9 180-199 47 47 47 79 79 79 F F F 10 200-219 47 47 47 79 79 79 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 45 45 45 77 77 77 F F F 15 300-319 45 45 45 77 77 77 F F F 16 320-339 44 44 44 76 76 76 F F F 17 340-359 44 44 44 75 75 75 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 42 42 42 70 70 70 F F F 20 400-419 47 47 47 79 79 79 F F F 21 420-439 47 47 47 79 79 79 F F F 22 440-459 46 46 46 78 78 78 F F F 23 460-479 47 47 47 79 79 79 F F F 24 480-499 42 42 42 70 70 70 F F F 25 500-519 46 46 46 78 78 78 F F F 26 520-539 46 46 46 78 78 78 F F F 27 540-559 46 46 46 78 78 78 F F F 28 560-579 43 43 43 71 71 71 F F F 29 580-599 46 46 46 78 78 78 F F F Max gen: 47, Max flush: 79 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.351130304Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.395625822Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.396464687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.396511291Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.396527093Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.396541444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382} {"msg":"Crucible 82bf55a2-5df5-48bf-980e-7cbf4f520636 has session id: 17e3b51c-b8e8-4582-8230-c66d252f2e59","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397232794Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Upstairs opts: Upstairs UUID: 82bf55a2-5df5-48bf-980e-7cbf4f520636, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397269258Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Crucible stats registered with UUID: 82bf55a2-5df5-48bf-980e-7cbf4f520636","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.39729393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:17.397315622Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397771088Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397796781Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382} {"msg":"82bf55a2-5df5-48bf-980e-7cbf4f520636 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397872389Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.39788965Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"0","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397913923Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"0","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397927814Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397942516Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.397959827Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"2","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.39798161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"2","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.398007732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.398033695Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.398059928Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"ds_connection connected from Ok(127.0.0.1:44001)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.398165228Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"ds_connection connected from Ok(127.0.0.1:48903)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.398199192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:17.398232655Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:17.398262788Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:17.398396851Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.398564379Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.398952138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"2","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.39907625Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"0","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.399551958Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"2","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.3996756Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"0","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.400249018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:17.400270251Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.400520566Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"ds_connection connected from Ok(127.0.0.1:34053)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.400649019Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.401616796Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.402309516Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[0]R flush_numbers[0..12]: [77, 70, 60, 78, 28, 78, 78, 79, 79, 79, 79, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.402956172Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[0]R generation[0..12]: [45, 42, 36, 46, 15, 46, 46, 47, 47, 47, 47, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.402977264Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.402992375Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[1]R flush_numbers[0..12]: [77, 70, 60, 78, 28, 78, 78, 79, 79, 79, 79, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403006346Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[1]R generation[0..12]: [45, 42, 36, 46, 15, 46, 46, 47, 47, 47, 47, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403020418Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403034149Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[2]R flush_numbers[0..12]: [77, 70, 60, 78, 28, 78, 78, 79, 79, 79, 79, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403047451Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[2]R generation[0..12]: [45, 42, 36, 46, 15, 46, 46, 47, 47, 47, 47, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403064852Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403078904Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403091385Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403109947Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403122928Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403135699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"":"downstairs","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403148311Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403159842Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"0","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403172693Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"1","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403185775Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"client":"2","":"downstairs","session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403198596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403210487Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"82bf55a2-5df5-48bf-980e-7cbf4f520636 is now active with session: 17e3b51c-b8e8-4582-8230-c66d252f2e59","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403222888Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.40323458Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382,"session_id":"17e3b51c-b8e8-4582-8230-c66d252f2e59"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:27.403246761Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 57 len 8192 data: 6 5 02/30 Write block 76 len 4096 data: 6 03/30 Read block 245 len 24576 04/30 Read block 524 len 40960 05/30 Write block 149 len 28672 data: 6 5 6 5 4 4 4 06/30 Read block 93 len 24576 07/30 Read block 133 len 32768 08/30 Read block 511 len 12288 09/30 Flush 10/30 Write block 210 len 40960 data: 4 3 3 3 3 4 4 4 3 4 11/30 Read block 476 len 20480 12/30 Read block 396 len 32768 13/30 Write block 393 len 20480 data: 8 7 8 8 7 14/30 Read block 42 len 20480 15/30 Read block 127 len 24576 16/30 Write block 46 len 24576 data: 5 5 4 4 4 4 17/30 Read block 28 len 40960 18/30 Write block 317 len 36864 data: 3 4 5 5 5 6 7 6 6 19/30 Flush 20/30 Write block 25 len 40960 data: 4 4 5 5 5 4 5 6 5 4 21/30 Write block 182 len 24576 data: 3 3 6 6 5 3 22/30 Flush 23/30 Write block 350 len 36864 data: 3 2 2 2 2 2 3 3 2 24/30 Read block 449 len 24576 25/30 Read block 80 len 16384 26/30 Flush 27/30 Write block 412 len 24576 data: 8 8 6 6 5 5 28/30 Read block 259 len 16384 29/30 Write block 78 len 36864 data: 5 6 6 5 3 3 3 3 5 30/30 Write block 519 len 24576 data: 8 7 6 7 5 5 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:27.425987204Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 77 77 77 F F F 1 020-039 48 48 48 82 82 82 F F F 2 040-059 48 48 48 81 81 81 F F F 3 060-079 48 48 48 80 80 80 T T T 4 080-099 15 15 15 28 28 28 T T T 5 100-119 46 46 46 78 78 78 F F F 6 120-139 46 46 46 78 78 78 F F F 7 140-159 48 48 48 80 80 80 F F F 8 160-179 47 47 47 79 79 79 F F F 9 180-199 48 48 48 82 82 82 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 45 45 45 77 77 77 F F F 15 300-319 48 48 48 81 81 81 F F F 16 320-339 48 48 48 81 81 81 F F F 17 340-359 48 48 48 83 83 83 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 48 48 48 81 81 81 F F F 20 400-419 47 47 47 79 79 79 T T T 21 420-439 47 47 47 79 79 79 F F F 22 440-459 46 46 46 78 78 78 F F F 23 460-479 47 47 47 79 79 79 F F F 24 480-499 42 42 42 70 70 70 F F F 25 500-519 46 46 46 78 78 78 T T T 26 520-539 46 46 46 78 78 78 T T T 27 540-559 46 46 46 78 78 78 F F F 28 560-579 43 43 43 71 71 71 F F F 29 580-599 46 46 46 78 78 78 F F F Max gen: 48, Max flush: 83 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 77 77 77 F F F 1 020-039 48 48 48 82 82 82 F F F 2 040-059 48 48 48 81 81 81 F F F 3 060-079 49 49 49 84 84 84 F F F 4 080-099 49 49 49 84 84 84 F F F 5 100-119 46 46 46 78 78 78 F F F 6 120-139 46 46 46 78 78 78 F F F 7 140-159 48 48 48 80 80 80 F F F 8 160-179 47 47 47 79 79 79 F F F 9 180-199 48 48 48 82 82 82 F F F 10 200-219 48 48 48 81 81 81 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 45 45 45 77 77 77 F F F 15 300-319 48 48 48 81 81 81 F F F 16 320-339 48 48 48 81 81 81 F F F 17 340-359 48 48 48 83 83 83 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 48 48 48 81 81 81 F F F 20 400-419 49 49 49 84 84 84 F F F 21 420-439 47 47 47 79 79 79 F F F 22 440-459 46 46 46 78 78 78 F F F 23 460-479 47 47 47 79 79 79 F F F 24 480-499 42 42 42 70 70 70 F F F 25 500-519 49 49 49 84 84 84 F F F 26 520-539 49 49 49 84 84 84 F F F 27 540-559 46 46 46 78 78 78 F F F 28 560-579 43 43 43 71 71 71 F F F 29 580-599 46 46 46 78 78 78 F F F Max gen: 49, Max flush: 84 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.666561256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:31.709581855Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.71002471Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.742023897Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.742853201Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.742912787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.742932839Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.74295074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390} {"msg":"Crucible 03831959-8015-4594-99d0-7efd0bffca27 has session id: 61451416-ce39-4c3f-aa82-6e7ada33e947","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.743645681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Upstairs opts: Upstairs UUID: 03831959-8015-4594-99d0-7efd0bffca27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.743675674Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Crucible stats registered with UUID: 03831959-8015-4594-99d0-7efd0bffca27","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.743690465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:31.743702807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.74413558Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744168013Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390} {"msg":"03831959-8015-4594-99d0-7efd0bffca27 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744223589Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744249562Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"0","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744274104Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"0","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744288426Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"1","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744302617Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"1","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744315878Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"2","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.7443301Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"2","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744343491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744357013Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744374454Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"ds_connection connected from Ok(127.0.0.1:53796)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744477225Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"ds_connection connected from Ok(127.0.0.1:35213)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744517409Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"ds_connection connected from Ok(127.0.0.1:35467)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.744684706Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.745359514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"1","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.745396207Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"2","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.745537212Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"0","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746003359Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"2","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746030141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"1","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746242943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"0","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746592258Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746657394Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[0]R flush_numbers[0..12]: [77, 82, 81, 84, 84, 78, 78, 80, 79, 82, 81, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746770816Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[0]R generation[0..12]: [45, 48, 48, 49, 49, 46, 46, 48, 47, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746791028Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.74680922Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[1]R flush_numbers[0..12]: [77, 82, 81, 84, 84, 78, 78, 80, 79, 82, 81, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746823461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[1]R generation[0..12]: [45, 48, 48, 49, 49, 46, 46, 48, 47, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746837012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746850704Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[2]R flush_numbers[0..12]: [77, 82, 81, 84, 84, 78, 78, 80, 79, 82, 81, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746863955Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[2]R generation[0..12]: [45, 48, 48, 49, 49, 46, 46, 48, 47, 48, 48, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746877367Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746890928Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:27:31.746903419Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746919901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746932352Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746944803Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"":"downstairs","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746957535Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746969026Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"0","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746981787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"1","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.746994588Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"client":"2","":"downstairs","session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.74700975Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.747021891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"03831959-8015-4594-99d0-7efd0bffca27 is now active with session: 61451416-ce39-4c3f-aa82-6e7ada33e947","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.747034543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.747046214Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390,"session_id":"61451416-ce39-4c3f-aa82-6e7ada33e947"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:31.747062295Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 559 len 16384 data: 6 4 4 3 03/30 Write block 1 len 40960 data: 3 4 6 6 5 5 5 5 4 4 04/30 Write block 300 len 20480 data: 8 7 8 8 9 05/30 Write block 176 len 24576 data: 5 2 2 3 3 3 06/30 Read block 456 len 8192 07/30 Read block 95 len 32768 08/30 Read block 277 len 8192 09/30 Read block 42 len 24576 10/30 Write block 425 len 28672 data: 5 5 6 5 3 3 2 11/30 Write block 4 len 28672 data: 7 6 6 6 6 5 5 12/30 Write block 210 len 28672 data: 5 4 4 4 4 5 5 13/30 Read block 33 len 4096 14/30 Write block 467 len 8192 data: 8 8 15/30 Read block 258 len 28672 16/30 Read block 147 len 36864 17/30 Read block 181 len 4096 18/30 Read block 279 len 36864 19/30 Write block 20 len 28672 data: 7 6 7 6 5 5 5 20/30 Flush 21/30 Write block 495 len 36864 data: 4 3 3 4 5 4 4 3 3 22/30 Read block 66 len 20480 23/30 Write block 363 len 8192 data: 2 2 24/30 Read block 587 len 28672 25/30 Write block 343 len 4096 data: 6 26/30 Write block 593 len 12288 data: 4 4 2 27/30 Write block 127 len 28672 data: 6 6 4 5 5 6 7 28/30 Write block 464 len 12288 data: 6 7 7 29/30 Read block 556 len 40960 30/30 Read block 307 len 8192 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:31.766274623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 50 50 50 86 86 86 F F F 2 040-059 48 48 48 81 81 81 F F F 3 060-079 49 49 49 84 84 84 F F F 4 080-099 49 49 49 84 84 84 F F F 5 100-119 46 46 46 78 78 78 F F F 6 120-139 46 46 46 78 78 78 T T T 7 140-159 48 48 48 80 80 80 F F F 8 160-179 50 50 50 86 86 86 F F F 9 180-199 50 50 50 86 86 86 F F F 10 200-219 50 50 50 86 86 86 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 45 45 45 77 77 77 F F F 15 300-319 50 50 50 86 86 86 F F F 16 320-339 48 48 48 81 81 81 F F F 17 340-359 48 48 48 83 83 83 T T T 18 360-379 39 39 39 65 65 65 T T T 19 380-399 48 48 48 81 81 81 F F F 20 400-419 49 49 49 84 84 84 F F F 21 420-439 50 50 50 86 86 86 F F F 22 440-459 46 46 46 78 78 78 F F F 23 460-479 50 50 50 86 86 86 T T T 24 480-499 42 42 42 70 70 70 T T T 25 500-519 49 49 49 84 84 84 T T T 26 520-539 49 49 49 84 84 84 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 46 46 46 78 78 78 T T T Max gen: 50, Max flush: 86 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 50 50 50 86 86 86 F F F 2 040-059 48 48 48 81 81 81 F F F 3 060-079 49 49 49 84 84 84 F F F 4 080-099 49 49 49 84 84 84 F F F 5 100-119 46 46 46 78 78 78 F F F 6 120-139 51 51 51 87 87 87 F F F 7 140-159 48 48 48 80 80 80 F F F 8 160-179 50 50 50 86 86 86 F F F 9 180-199 50 50 50 86 86 86 F F F 10 200-219 50 50 50 86 86 86 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 45 45 45 77 77 77 F F F 15 300-319 50 50 50 86 86 86 F F F 16 320-339 48 48 48 81 81 81 F F F 17 340-359 51 51 51 87 87 87 F F F 18 360-379 51 51 51 87 87 87 F F F 19 380-399 48 48 48 81 81 81 F F F 20 400-419 49 49 49 84 84 84 F F F 21 420-439 50 50 50 86 86 86 F F F 22 440-459 46 46 46 78 78 78 F F F 23 460-479 51 51 51 87 87 87 F F F 24 480-499 51 51 51 87 87 87 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 49 49 49 84 84 84 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 51 51 51 87 87 87 F F F Max gen: 51, Max flush: 87 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.045798973Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:36.089347965Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.089849786Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.121494227Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.122229101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.122265725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.122280126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.122293968Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398} {"msg":"Crucible 9e3ec49d-c180-4942-8eba-027a4c4aaf16 has session id: 297951d2-ae89-4a2f-9d12-3c1428849ae2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.122900129Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Upstairs opts: Upstairs UUID: 9e3ec49d-c180-4942-8eba-027a4c4aaf16, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.122938453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Crucible stats registered with UUID: 9e3ec49d-c180-4942-8eba-027a4c4aaf16","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.122964055Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:36.122984937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123439423Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123461296Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398} {"msg":"9e3ec49d-c180-4942-8eba-027a4c4aaf16 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123536773Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123553145Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"0","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123579117Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"0","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123593179Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"1","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.12360723Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"1","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123620332Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"2","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123634203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"2","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123647034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123660436Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123673567Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53200)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123781018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59444)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123822632Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:33072)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.123849295Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.124510231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"2","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.124538014Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"1","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.124637244Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"0","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.12518911Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"1","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.125232644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"0","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.125263847Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"2","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.125796971Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.125920213Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[0]R flush_numbers[0..12]: [86, 86, 81, 84, 84, 78, 87, 80, 86, 86, 86, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.125958317Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[0]R generation[0..12]: [50, 50, 48, 49, 49, 46, 51, 48, 50, 50, 50, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.125974759Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.12598944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[1]R flush_numbers[0..12]: [86, 86, 81, 84, 84, 78, 87, 80, 86, 86, 86, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126003612Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[1]R generation[0..12]: [50, 50, 48, 49, 49, 46, 51, 48, 50, 50, 50, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126016993Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126030425Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[2]R flush_numbers[0..12]: [86, 86, 81, 84, 84, 78, 87, 80, 86, 86, 86, 68]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126044536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[2]R generation[0..12]: [50, 50, 48, 49, 49, 46, 51, 48, 50, 50, 50, 40]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126057707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126071159Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.12608394Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126101132Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126117123Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126140016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"":"downstairs","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126171379Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126194691Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"0","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126216583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"1","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126230345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"client":"2","":"downstairs","session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126243196Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126263018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"9e3ec49d-c180-4942-8eba-027a4c4aaf16 is now active with session: 297951d2-ae89-4a2f-9d12-3c1428849ae2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.1262858Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126308943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398,"session_id":"297951d2-ae89-4a2f-9d12-3c1428849ae2"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:36.126329675Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 225 len 36864 data: 7 7 5 5 4 5 6 5 5 02/30 Read block 228 len 40960 03/30 Read block 315 len 36864 04/30 Read block 322 len 4096 05/30 Read block 404 len 40960 06/30 Read block 118 len 20480 07/30 Read block 190 len 12288 08/30 Read block 17 len 40960 09/30 Write block 588 len 28672 data: 3 3 4 5 4 5 5 10/30 Write block 83 len 32768 data: 4 4 4 6 5 4 4 4 11/30 Write block 439 len 32768 data: 9 7 6 5 4 3 2 2 12/30 Write block 291 len 36864 data: 4 4 4 5 5 5 6 6 6 13/30 Write block 521 len 20480 data: 7 8 6 6 5 14/30 Write block 526 len 16384 data: 5 5 5 7 15/30 Read block 254 len 20480 16/30 Write block 75 len 4096 data: 6 17/30 Write block 149 len 28672 data: 7 6 7 6 5 5 5 18/30 Read block 222 len 16384 19/30 Read block 5 len 24576 20/30 Read block 11 len 12288 21/30 Write block 116 len 40960 data: 8 10 10 10 11 10 7 7 7 7 22/30 Read block 107 len 36864 23/30 Write block 313 len 12288 data: 3 2 2 24/30 Read block 36 len 40960 25/30 Read block 148 len 28672 26/30 Write block 446 len 36864 data: 3 2 2 2 2 2 5 5 5 27/30 Write block 85 len 24576 data: 5 7 6 5 5 5 28/30 Read block 387 len 4096 29/30 Write block 590 len 16384 data: 5 6 5 6 30/30 Write block 357 len 36864 data: 4 3 2 2 2 2 3 3 3 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:36.147724043Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 50 50 50 86 86 86 F F F 2 040-059 48 48 48 81 81 81 F F F 3 060-079 49 49 49 84 84 84 T T T 4 080-099 49 49 49 84 84 84 T T T 5 100-119 46 46 46 78 78 78 T T T 6 120-139 51 51 51 87 87 87 T T T 7 140-159 48 48 48 80 80 80 T T T 8 160-179 50 50 50 86 86 86 F F F 9 180-199 50 50 50 86 86 86 F F F 10 200-219 50 50 50 86 86 86 F F F 11 220-239 40 40 40 68 68 68 T T T 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 45 45 45 77 77 77 T T T 15 300-319 50 50 50 86 86 86 T T T 16 320-339 48 48 48 81 81 81 F F F 17 340-359 51 51 51 87 87 87 T T T 18 360-379 51 51 51 87 87 87 T T T 19 380-399 48 48 48 81 81 81 F F F 20 400-419 49 49 49 84 84 84 F F F 21 420-439 50 50 50 86 86 86 T T T 22 440-459 46 46 46 78 78 78 T T T 23 460-479 51 51 51 87 87 87 F F F 24 480-499 51 51 51 87 87 87 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 49 49 49 84 84 84 T T T 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 51 51 51 87 87 87 T T T Max gen: 51, Max flush: 87 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 50 50 50 86 86 86 F F F 2 040-059 48 48 48 81 81 81 F F F 3 060-079 52 52 52 88 88 88 F F F 4 080-099 52 52 52 88 88 88 F F F 5 100-119 52 52 52 88 88 88 F F F 6 120-139 52 52 52 88 88 88 F F F 7 140-159 52 52 52 88 88 88 F F F 8 160-179 50 50 50 86 86 86 F F F 9 180-199 50 50 50 86 86 86 F F F 10 200-219 50 50 50 86 86 86 F F F 11 220-239 52 52 52 88 88 88 F F F 12 240-259 44 44 44 74 74 74 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 52 52 52 88 88 88 F F F 16 320-339 48 48 48 81 81 81 F F F 17 340-359 52 52 52 88 88 88 F F F 18 360-379 52 52 52 88 88 88 F F F 19 380-399 48 48 48 81 81 81 F F F 20 400-419 49 49 49 84 84 84 F F F 21 420-439 52 52 52 88 88 88 F F F 22 440-459 52 52 52 88 88 88 F F F 23 460-479 51 51 51 87 87 87 F F F 24 480-499 51 51 51 87 87 87 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 52 52 52 88 88 88 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 52, Max flush: 88 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.473147412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.517692134Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.51853627Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.518591765Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.518607337Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.518626829Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406} {"msg":"Crucible 5ef33aab-effe-4003-9169-4615dfb9dbe1 has session id: c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519163933Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Upstairs opts: Upstairs UUID: 5ef33aab-effe-4003-9169-4615dfb9dbe1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519199357Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Crucible stats registered with UUID: 5ef33aab-effe-4003-9169-4615dfb9dbe1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519215158Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:40.51922792Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.51973045Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519763514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406} {"msg":"5ef33aab-effe-4003-9169-4615dfb9dbe1 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519815999Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519845142Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"0","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519882935Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"0","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519908298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"1","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519935451Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"1","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519960943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.519987006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.520012688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.520035901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.520096697Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:40.520132141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:40.520197587Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:40.52022732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51284)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.520249592Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.520309118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:58343)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.520347902Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.520724Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"1","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.520857324Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"0","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.521285837Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"1","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.5214153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"0","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.521866006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:40.521971956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.521037802Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:41586)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.521128491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.522215741Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.522814431Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[0]R flush_numbers[0..12]: [86, 86, 81, 88, 88, 88, 88, 88, 86, 86, 86, 88]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523433264Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[0]R generation[0..12]: [50, 50, 48, 52, 52, 52, 52, 52, 50, 50, 50, 52]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523453856Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523469087Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[1]R flush_numbers[0..12]: [86, 86, 81, 88, 88, 88, 88, 88, 86, 86, 86, 88]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523482789Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[1]R generation[0..12]: [50, 50, 48, 52, 52, 52, 52, 52, 50, 50, 50, 52]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.52349617Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523510221Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[2]R flush_numbers[0..12]: [86, 86, 81, 88, 88, 88, 88, 88, 86, 86, 86, 88]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523523643Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[2]R generation[0..12]: [50, 50, 48, 52, 52, 52, 52, 52, 50, 50, 50, 52]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523536894Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523550826Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Max found gen is 53","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:27:50.523563687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523582839Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.52359573Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523609191Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"":"downstairs","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523622273Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523633824Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"0","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523647195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"1","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523660426Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"client":"2","":"downstairs","session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523673708Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "time":"2025-02-27T17:27:50.523685599Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"5ef33aab-effe-4003-9169-4615dfb9dbe1 is now active with session: c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523701861Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523713522Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406,"session_id":"c9ef9af6-dc1f-4984-a0b5-2e6871cb35fb"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:27:50.523725483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 538 len 8192 data: 5 5 02/30 Write block 417 len 28672 data: 6 6 5 3 4 4 4 03/30 Write block 173 len 8192 data: 4 5 04/30 Write block 109 len 16384 data: 7 8 6 7 05/30 Read block 119 len 24576 06/30 Read block 194 len 4096 07/30 Write block 59 len 20480 data: 4 4 5 4 5 08/30 Flush 09/30 Write block 166 len 20480 data: 3 3 3 3 3 10/30 Read block 401 len 32768 11/30 Write block 311 len 8192 data: 4 4 12/30 Flush 13/30 Write block 235 len 28672 data: 6 7 5 5 5 3 3 14/30 Write block 52 len 16384 data: 4 4 6 6 15/30 Flush 16/30 Write block 444 len 4096 data: 4 17/30 Write block 318 len 16384 data: 5 6 6 6 18/30 Flush 19/30 Write block 410 len 20480 data: 8 5 9 9 7 20/30 Read block 34 len 8192 21/30 Read block 41 len 16384 22/30 Write block 177 len 8192 data: 3 3 23/30 Read block 94 len 28672 24/30 Flush 25/30 Write block 236 len 8192 data: 8 6 26/30 Flush 27/30 Write block 380 len 40960 data: 3 2 3 5 5 8 7 6 7 6 28/30 Write block 369 len 24576 data: 2 2 4 4 3 3 29/30 Write block 83 len 28672 data: 5 5 6 8 7 6 6 30/30 Write block 122 len 12288 data: 8 8 8 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:27:50.540608186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 50 50 50 86 86 86 F F F 2 040-059 54 54 54 91 91 91 F F F 3 060-079 54 54 54 89 89 89 F F F 4 080-099 52 52 52 88 88 88 T T T 5 100-119 54 54 54 89 89 89 F F F 6 120-139 52 52 52 88 88 88 T T T 7 140-159 52 52 52 88 88 88 F F F 8 160-179 54 54 54 93 93 93 F F F 9 180-199 50 50 50 86 86 86 F F F 10 200-219 50 50 50 86 86 86 F F F 11 220-239 54 54 54 94 94 94 F F F 12 240-259 54 54 54 91 91 91 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 54 54 54 92 92 92 F F F 16 320-339 54 54 54 92 92 92 F F F 17 340-359 52 52 52 88 88 88 F F F 18 360-379 52 52 52 88 88 88 T T T 19 380-399 48 48 48 81 81 81 T T T 20 400-419 54 54 54 93 93 93 F F F 21 420-439 54 54 54 89 89 89 F F F 22 440-459 54 54 54 92 92 92 F F F 23 460-479 51 51 51 87 87 87 F F F 24 480-499 51 51 51 87 87 87 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 54, Max flush: 94 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 50 50 50 86 86 86 F F F 2 040-059 54 54 54 91 91 91 F F F 3 060-079 54 54 54 89 89 89 F F F 4 080-099 55 55 55 95 95 95 F F F 5 100-119 54 54 54 89 89 89 F F F 6 120-139 55 55 55 95 95 95 F F F 7 140-159 52 52 52 88 88 88 F F F 8 160-179 54 54 54 93 93 93 F F F 9 180-199 50 50 50 86 86 86 F F F 10 200-219 50 50 50 86 86 86 F F F 11 220-239 54 54 54 94 94 94 F F F 12 240-259 54 54 54 91 91 91 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 54 54 54 92 92 92 F F F 16 320-339 54 54 54 92 92 92 F F F 17 340-359 52 52 52 88 88 88 F F F 18 360-379 55 55 55 95 95 95 F F F 19 380-399 55 55 55 95 95 95 F F F 20 400-419 54 54 54 93 93 93 F F F 21 420-439 54 54 54 89 89 89 F F F 22 440-459 54 54 54 92 92 92 F F F 23 460-479 51 51 51 87 87 87 F F F 24 480-499 51 51 51 87 87 87 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 50 50 50 86 86 86 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 55, Max flush: 95 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.780572455Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:04.823533999Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.823954881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.855458358Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.856252028Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.856287572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.856301663Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.856315675Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414} {"msg":"Crucible 94376fdb-12ee-4c72-aee7-70250c261aa5 has session id: 67a961cc-2ef4-4ff0-b6ff-f052f4603e77","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.856939268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Upstairs opts: Upstairs UUID: 94376fdb-12ee-4c72-aee7-70250c261aa5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.856968891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Crucible stats registered with UUID: 94376fdb-12ee-4c72-aee7-70250c261aa5","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.856988533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:04.857018076Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857373461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857404845Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414} {"msg":"94376fdb-12ee-4c72-aee7-70250c261aa5 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.85746006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857485513Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"0","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857510215Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"0","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857534468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"1","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.85756106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"1","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857586843Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"2","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857615866Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"2","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857642749Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857670162Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857696444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"ds_connection connected from Ok(127.0.0.1:38841)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857723437Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"ds_connection connected from Ok(127.0.0.1:39035)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857763911Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"ds_connection connected from Ok(127.0.0.1:61392)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.857796574Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.85844812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"1","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.858486264Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"2","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.858537569Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"0","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.859175223Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"1","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.859214837Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"2","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.859325348Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"0","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.859855552Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.859892386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[0]R flush_numbers[0..12]: [86, 86, 91, 89, 95, 89, 95, 88, 93, 86, 86, 94]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.859956142Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[0]R generation[0..12]: [50, 50, 54, 54, 55, 54, 55, 52, 54, 50, 50, 54]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.859981044Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.859995666Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[1]R flush_numbers[0..12]: [86, 86, 91, 89, 95, 89, 95, 88, 93, 86, 86, 94]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860010207Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[1]R generation[0..12]: [50, 50, 54, 54, 55, 54, 55, 52, 54, 50, 50, 54]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860024039Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860041911Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[2]R flush_numbers[0..12]: [86, 86, 91, 89, 95, 89, 95, 88, 93, 86, 86, 94]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860055572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[2]R generation[0..12]: [50, 50, 54, 54, 55, 54, 55, 52, 54, 50, 50, 54]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860068923Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860082625Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860095446Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860115268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860127799Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.86014036Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"":"downstairs","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860153382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860165263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"0","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860178154Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"1","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860190946Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"client":"2","":"downstairs","session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860219928Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.86023356Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"94376fdb-12ee-4c72-aee7-70250c261aa5 is now active with session: 67a961cc-2ef4-4ff0-b6ff-f052f4603e77","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860247361Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860261443Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414,"session_id":"67a961cc-2ef4-4ff0-b6ff-f052f4603e77"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:04.860273454Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 27 len 16384 data: 6 6 6 5 02/30 Read block 376 len 8192 03/30 Flush 04/30 Write block 450 len 16384 data: 3 3 6 6 05/30 Write block 187 len 20480 data: 4 3 2 2 3 06/30 Write block 403 len 8192 data: 2 3 07/30 Write block 464 len 36864 data: 7 8 8 9 9 8 8 8 6 08/30 Read block 447 len 4096 09/30 Write block 420 len 12288 data: 4 5 5 10/30 Read block 213 len 8192 11/30 Flush 12/30 Write block 377 len 40960 data: 3 3 4 4 3 4 6 6 9 8 13/30 Read block 27 len 32768 14/30 Write block 205 len 20480 data: 5 5 4 3 4 15/30 Flush 16/30 Write block 153 len 24576 data: 6 6 6 4 5 5 17/30 Read block 95 len 36864 18/30 Write block 471 len 8192 data: 9 7 19/30 Write block 552 len 32768 data: 5 5 3 3 4 5 5 7 20/30 Read block 158 len 40960 21/30 Read block 521 len 32768 22/30 Write block 170 len 12288 data: 4 4 4 23/30 Read block 114 len 8192 24/30 Write block 315 len 20480 data: 3 2 4 6 7 25/30 Read block 253 len 12288 26/30 Write block 225 len 4096 data: 8 27/30 Read block 208 len 4096 28/30 Write block 462 len 24576 data: 5 5 8 9 9 10 29/30 Read block 330 len 12288 30/30 Write block 347 len 28672 data: 2 3 4 4 3 3 3 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:04.879715945Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 56 56 56 96 96 96 F F F 2 040-059 54 54 54 91 91 91 F F F 3 060-079 54 54 54 89 89 89 F F F 4 080-099 55 55 55 95 95 95 F F F 5 100-119 54 54 54 89 89 89 F F F 6 120-139 55 55 55 95 95 95 F F F 7 140-159 52 52 52 88 88 88 T T T 8 160-179 54 54 54 93 93 93 T T T 9 180-199 56 56 56 97 97 97 F F F 10 200-219 56 56 56 98 98 98 F F F 11 220-239 54 54 54 94 94 94 T T T 12 240-259 54 54 54 91 91 91 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 54 54 54 92 92 92 T T T 16 320-339 54 54 54 92 92 92 F F F 17 340-359 52 52 52 88 88 88 T T T 18 360-379 56 56 56 98 98 98 F F F 19 380-399 56 56 56 98 98 98 F F F 20 400-419 56 56 56 97 97 97 F F F 21 420-439 56 56 56 97 97 97 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 56 56 56 97 97 97 T T T 24 480-499 51 51 51 87 87 87 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 50 50 50 86 86 86 T T T 28 560-579 50 50 50 86 86 86 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 56, Max flush: 98 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 56 56 56 96 96 96 F F F 2 040-059 54 54 54 91 91 91 F F F 3 060-079 54 54 54 89 89 89 F F F 4 080-099 55 55 55 95 95 95 F F F 5 100-119 54 54 54 89 89 89 F F F 6 120-139 55 55 55 95 95 95 F F F 7 140-159 57 57 57 99 99 99 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 56 56 56 97 97 97 F F F 10 200-219 56 56 56 98 98 98 F F F 11 220-239 57 57 57 99 99 99 F F F 12 240-259 54 54 54 91 91 91 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 99 99 99 F F F 16 320-339 54 54 54 92 92 92 F F F 17 340-359 57 57 57 99 99 99 F F F 18 360-379 56 56 56 98 98 98 F F F 19 380-399 56 56 56 98 98 98 F F F 20 400-419 56 56 56 97 97 97 F F F 21 420-439 56 56 56 97 97 97 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 57 57 57 99 99 99 F F F 24 480-499 51 51 51 87 87 87 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 57 57 57 99 99 99 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 52 52 52 88 88 88 F F F Max gen: 57, Max flush: 99 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.159666618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:09.202683107Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.203151254Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.234873373Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.235569533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.235609517Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.235625849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.235643361Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422} {"msg":"Crucible ae4d41cf-fe6c-42c0-9c01-2681b7be0781 has session id: 2f5a524f-da06-4f41-9ede-79624e7b2d42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.236321819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Upstairs opts: Upstairs UUID: ae4d41cf-fe6c-42c0-9c01-2681b7be0781, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.236349252Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Crucible stats registered with UUID: ae4d41cf-fe6c-42c0-9c01-2681b7be0781","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.236363703Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:09.236376135Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.236861574Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.236884716Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422} {"msg":"ae4d41cf-fe6c-42c0-9c01-2681b7be0781 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.236944312Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.236970435Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"0","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237002188Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"0","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.23702818Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"1","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237061694Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"1","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237087206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"2","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237108908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"2","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237132111Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237159054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237183446Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:48071)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237207738Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:57699)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237230211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:43754)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.237254693Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.238083957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"2","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.238133362Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"1","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.23830848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"0","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.238868806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"1","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.238957345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"2","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239152125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"0","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239558556Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239588809Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[0]R flush_numbers[0..12]: [86, 96, 91, 89, 95, 89, 95, 99, 99, 97, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239713721Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[0]R generation[0..12]: [50, 56, 54, 54, 55, 54, 55, 57, 57, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239733833Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239748585Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[1]R flush_numbers[0..12]: [86, 96, 91, 89, 95, 89, 95, 99, 99, 97, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239762356Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[1]R generation[0..12]: [50, 56, 54, 54, 55, 54, 55, 57, 57, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239775777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239789199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[2]R flush_numbers[0..12]: [86, 96, 91, 89, 95, 89, 95, 99, 99, 97, 98, 99]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.23980222Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[2]R generation[0..12]: [50, 56, 54, 54, 55, 54, 55, 57, 57, 56, 56, 57]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239815301Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239828693Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239841394Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:28:09.239854216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239870027Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239885729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"":"downstairs","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.23989892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239910581Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"0","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239923492Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"1","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239936224Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"client":"2","":"downstairs","session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239948955Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239960876Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"ae4d41cf-fe6c-42c0-9c01-2681b7be0781 is now active with session: 2f5a524f-da06-4f41-9ede-79624e7b2d42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239973317Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.239984799Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422,"session_id":"2f5a524f-da06-4f41-9ede-79624e7b2d42"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:09.23999652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 597 len 8192 data: 2 3 02/30 Read block 498 len 28672 03/30 Flush 04/30 Write block 461 len 8192 data: 7 6 05/30 Read block 192 len 24576 06/30 Write block 481 len 36864 data: 3 3 4 4 4 4 4 6 6 07/30 Write block 406 len 40960 data: 5 5 8 8 9 6 10 10 8 7 08/30 Read block 228 len 40960 09/30 Read block 285 len 32768 10/30 Read block 474 len 24576 11/30 Write block 429 len 20480 data: 4 4 3 2 2 12/30 Read block 569 len 28672 13/30 Write block 109 len 40960 data: 8 9 7 8 6 6 6 9 11 11 14/30 Read block 346 len 36864 15/30 Write block 348 len 4096 data: 4 16/30 Flush 17/30 Write block 387 len 16384 data: 7 8 7 6 18/30 Read block 567 len 40960 19/30 Read block 301 len 12288 20/30 Read block 556 len 32768 21/30 Write block 98 len 40960 data: 2 2 2 2 3 4 4 6 6 6 22/30 Write block 253 len 4096 data: 11 23/30 Read block 452 len 4096 24/30 Flush 25/30 Write block 317 len 4096 data: 5 26/30 Write block 310 len 32768 data: 5 5 5 4 3 4 3 6 27/30 Write block 218 len 20480 data: 4 5 4 4 6 28/30 Write block 41 len 16384 data: 5 5 6 4 29/30 Write block 86 len 36864 data: 9 8 7 7 6 4 4 4 3 30/30 Write block 541 len 12288 data: 6 7 7 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 5 Sent Done Sent false 1027 Acked Write 4 Sent Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 2 4 10 Done 2 4 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:09.259462303Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 56 56 56 96 96 96 F F F 2 040-059 54 54 54 91 91 91 T T T 3 060-079 54 54 54 89 89 89 F F F 4 080-099 58 58 58 102 102 102 T T T 5 100-119 58 58 58 102 102 102 F F F 6 120-139 55 55 55 95 95 95 F F F 7 140-159 57 57 57 99 99 99 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 56 56 56 97 97 97 F F F 10 200-219 56 56 56 98 98 98 T T T 11 220-239 57 57 57 99 99 99 T T T 12 240-259 58 58 58 102 102 102 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 99 99 99 T T T 16 320-339 54 54 54 92 92 92 F F F 17 340-359 58 58 58 101 101 101 F F F 18 360-379 56 56 56 98 98 98 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 58 58 58 101 101 101 F F F 21 420-439 58 58 58 101 101 101 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 57 57 57 99 99 99 T T T 28 560-579 50 50 50 86 86 86 F F F 29 580-599 58 58 58 100 100 100 F F F Max gen: 58, Max flush: 102 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 56 56 56 96 96 96 F F F 2 040-059 59 59 59 103 103 103 F F F 3 060-079 54 54 54 89 89 89 F F F 4 080-099 59 59 59 103 103 103 F F F 5 100-119 58 58 58 102 102 102 F F F 6 120-139 55 55 55 95 95 95 F F F 7 140-159 57 57 57 99 99 99 F F F 8 160-179 57 57 57 99 99 99 F F F 9 180-199 56 56 56 97 97 97 F F F 10 200-219 59 59 59 103 103 103 F F F 11 220-239 59 59 59 103 103 103 F F F 12 240-259 58 58 58 102 102 102 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 59 59 59 103 103 103 F F F 16 320-339 54 54 54 92 92 92 F F F 17 340-359 58 58 58 101 101 101 F F F 18 360-379 56 56 56 98 98 98 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 58 58 58 101 101 101 F F F 21 420-439 58 58 58 101 101 101 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 59 59 59 103 103 103 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 58 58 58 100 100 100 F F F Max gen: 59, Max flush: 103 Begin loop 30 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.538028957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:13.583163509Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.583678121Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.615231063Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.615910691Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.615946285Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.615959496Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.615973488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430} {"msg":"Crucible 1f5ab00a-7204-4394-90b9-bd4983c260bf has session id: a84442bf-213c-4cd4-a7fd-a4ba48ba6a43","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.616654016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Upstairs opts: Upstairs UUID: 1f5ab00a-7204-4394-90b9-bd4983c260bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.61669091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Crucible stats registered with UUID: 1f5ab00a-7204-4394-90b9-bd4983c260bf","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.616717393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:13.616735905Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617247776Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617274199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430} {"msg":"1f5ab00a-7204-4394-90b9-bd4983c260bf active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617353557Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"0","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617409352Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"0","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617423974Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"1","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617437946Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"1","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617450947Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"2","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617464708Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"2","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617478739Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617492211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617507982Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"ds_connection connected from Ok(127.0.0.1:52111)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617640216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"ds_connection connected from Ok(127.0.0.1:55652)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.61767751Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"ds_connection connected from Ok(127.0.0.1:50705)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.617704642Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.618443527Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"1","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.618483961Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"0","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.618514024Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"2","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619130476Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"2","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619158319Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"0","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.61917369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"1","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619798633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619823806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[0]R flush_numbers[0..12]: [86, 96, 103, 89, 103, 102, 95, 99, 99, 97, 103, 103]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619893583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[0]R generation[0..12]: [50, 56, 59, 54, 59, 58, 55, 57, 57, 56, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619912255Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619926837Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[1]R flush_numbers[0..12]: [86, 96, 103, 89, 103, 102, 95, 99, 99, 97, 103, 103]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619940598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[1]R generation[0..12]: [50, 56, 59, 54, 59, 58, 55, 57, 57, 56, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619953699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.61996755Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[2]R flush_numbers[0..12]: [86, 96, 103, 89, 103, 102, 95, 99, 99, 97, 103, 103]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619984922Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[2]R generation[0..12]: [50, 56, 59, 54, 59, 58, 55, 57, 57, 56, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.619998704Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:28:13.620012875Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620030227Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620042888Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620055819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620068431Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"":"downstairs","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620081322Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620093153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"0","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620106004Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"1","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620118606Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"client":"2","":"downstairs","session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620131527Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620143398Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"1f5ab00a-7204-4394-90b9-bd4983c260bf is now active with session: a84442bf-213c-4cd4-a7fd-a4ba48ba6a43","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620155869Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620167291Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430,"session_id":"a84442bf-213c-4cd4-a7fd-a4ba48ba6a43"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:13.620179052Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 34 len 36864 data: 5 5 5 6 6 7 6 6 6 02/30 Write block 222 len 36864 data: 7 6 6 9 8 6 6 5 6 03/30 Write block 89 len 16384 data: 8 7 5 5 04/30 Read block 290 len 8192 05/30 Write block 224 len 4096 data: 7 06/30 Read block 216 len 36864 07/30 Read block 212 len 24576 08/30 Read block 545 len 20480 09/30 Read block 153 len 24576 10/30 Read block 558 len 20480 11/30 Read block 8 len 16384 12/30 Write block 174 len 16384 data: 6 5 6 4 13/30 Read block 66 len 28672 14/30 Read block 263 len 36864 15/30 Read block 450 len 28672 16/30 Write block 192 len 16384 data: 3 4 4 5 17/30 Write block 85 len 36864 data: 7 10 9 8 9 8 6 6 5 18/30 Read block 423 len 4096 19/30 Read block 85 len 36864 20/30 Write block 327 len 24576 data: 5 5 5 7 6 8 21/30 Read block 582 len 4096 22/30 Read block 452 len 20480 23/30 Read block 199 len 12288 24/30 Read block 423 len 40960 25/30 Write block 256 len 24576 data: 3 4 4 5 4 5 26/30 Write block 285 len 28672 data: 3 4 5 4 4 4 5 27/30 Write block 125 len 4096 data: 8 28/30 Read block 524 len 8192 29/30 Write block 545 len 36864 data: 7 7 8 8 8 9 7 6 6 30/30 Read block 168 len 12288 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 29 30 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:13.641433546Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 56 56 56 96 96 96 T T T 2 040-059 59 59 59 103 103 103 T T T 3 060-079 54 54 54 89 89 89 F F F 4 080-099 59 59 59 103 103 103 T T T 5 100-119 58 58 58 102 102 102 F F F 6 120-139 55 55 55 95 95 95 T T T 7 140-159 57 57 57 99 99 99 F F F 8 160-179 57 57 57 99 99 99 T T T 9 180-199 56 56 56 97 97 97 T T T 10 200-219 59 59 59 103 103 103 F F F 11 220-239 59 59 59 103 103 103 T T T 12 240-259 58 58 58 102 102 102 T T T 13 260-279 42 42 42 70 70 70 T T T 14 280-299 52 52 52 88 88 88 T T T 15 300-319 59 59 59 103 103 103 F F F 16 320-339 54 54 54 92 92 92 T T T 17 340-359 58 58 58 101 101 101 F F F 18 360-379 56 56 56 98 98 98 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 58 58 58 101 101 101 F F F 21 420-439 58 58 58 101 101 101 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 59 59 59 103 103 103 T T T 28 560-579 50 50 50 86 86 86 F F F 29 580-599 58 58 58 100 100 100 F F F Max gen: 59, Max flush: 103 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 86 86 86 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 60 60 60 104 104 104 F F F 3 060-079 54 54 54 89 89 89 F F F 4 080-099 60 60 60 104 104 104 F F F 5 100-119 58 58 58 102 102 102 F F F 6 120-139 60 60 60 104 104 104 F F F 7 140-159 57 57 57 99 99 99 F F F 8 160-179 60 60 60 104 104 104 F F F 9 180-199 60 60 60 104 104 104 F F F 10 200-219 59 59 59 103 103 103 F F F 11 220-239 60 60 60 104 104 104 F F F 12 240-259 60 60 60 104 104 104 F F F 13 260-279 60 60 60 104 104 104 F F F 14 280-299 60 60 60 104 104 104 F F F 15 300-319 59 59 59 103 103 103 F F F 16 320-339 60 60 60 104 104 104 F F F 17 340-359 58 58 58 101 101 101 F F F 18 360-379 56 56 56 98 98 98 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 58 58 58 101 101 101 F F F 21 420-439 58 58 58 101 101 101 F F F 22 440-459 56 56 56 97 97 97 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 50 50 50 86 86 86 F F F 29 580-599 58 58 58 100 100 100 F F F Max gen: 60, Max flush: 104 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.953149102Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.997483304Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.998225099Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.998270753Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.998284905Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.998298856Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438} {"msg":"Crucible 87e70f80-e86c-4b77-912c-4709a0a5dc37 has session id: 5eb40eee-6173-4b7e-982c-cc03f1b2b492","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.998922839Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Upstairs opts: Upstairs UUID: 87e70f80-e86c-4b77-912c-4709a0a5dc37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.998961503Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Crucible stats registered with UUID: 87e70f80-e86c-4b77-912c-4709a0a5dc37","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.998985045Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:17.999001697Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999384796Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999420269Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438} {"msg":"87e70f80-e86c-4b77-912c-4709a0a5dc37 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999491786Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999508528Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"0","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999536721Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"0","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999563514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"1","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999664564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"1","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999708468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"2","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999735671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"2","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999761954Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999788156Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999816039Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"ds_connection connected from Ok(127.0.0.1:63438)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999842762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"ds_connection connected from Ok(127.0.0.1:42981)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999873715Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"ds_connection connected from Ok(127.0.0.1:58389)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:17.999906658Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.000545233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"1","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.000787767Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"0","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.001304429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"1","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.001450534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"0","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.001957985Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.002065216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.002893839Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"2","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.003411962Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"2","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[0]R flush_numbers[0..12]: [86, 104, 104, 89, 104, 102, 104, 99, 104, 104, 103, 104]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.003923823Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[0]R generation[0..12]: [50, 60, 60, 54, 60, 58, 60, 57, 60, 60, 59, 60]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.003943775Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.003958747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[1]R flush_numbers[0..12]: [86, 104, 104, 89, 104, 102, 104, 99, 104, 104, 103, 104]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.003973568Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[1]R generation[0..12]: [50, 60, 60, 54, 60, 58, 60, 57, 60, 60, 59, 60]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.00398712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004009492Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[2]R flush_numbers[0..12]: [86, 104, 104, 89, 104, 102, 104, 99, 104, 104, 103, 104]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004027184Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[2]R generation[0..12]: [50, 60, 60, 54, 60, 58, 60, 57, 60, 60, 59, 60]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004040955Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004054567Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Max found gen is 61","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004073818Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Generation requested: 62 >= found:61","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.00408652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004099091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004114613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"":"downstairs","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004127714Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004139445Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"0","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004152206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"1","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004165127Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"client":"2","":"downstairs","session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004177949Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.00418986Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"87e70f80-e86c-4b77-912c-4709a0a5dc37 is now active with session: 5eb40eee-6173-4b7e-982c-cc03f1b2b492","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004202451Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004214092Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438,"session_id":"5eb40eee-6173-4b7e-982c-cc03f1b2b492"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:18.004225924Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 13 len 20480 data: 4 4 7 7 6 02/30 Read block 491 len 20480 03/30 Write block 140 len 36864 data: 10 9 8 7 4 3 2 3 3 04/30 Flush 05/30 Flush 06/30 Write block 302 len 20480 data: 9 9 10 6 6 07/30 Write block 316 len 40960 data: 4 7 7 8 7 7 7 8 7 7 08/30 Read block 241 len 16384 09/30 Write block 347 len 8192 data: 3 5 10/30 Write block 413 len 20480 data: 11 9 8 6 7 11/30 Write block 530 len 36864 data: 7 6 4 5 4 4 4 3 6 12/30 Read block 237 len 28672 13/30 Read block 305 len 16384 14/30 Read block 284 len 4096 15/30 Write block 140 len 40960 data: 11 10 9 8 5 4 3 4 4 8 16/30 Write block 544 len 4096 data: 6 17/30 Read block 59 len 36864 18/30 Write block 570 len 24576 data: 3 3 4 5 5 6 19/30 Read block 505 len 28672 20/30 Write block 456 len 12288 data: 5 5 6 21/30 Read block 117 len 32768 22/30 Write block 55 len 32768 data: 7 6 7 6 5 5 6 5 23/30 Write block 120 len 20480 data: 12 11 9 9 9 24/30 Write block 441 len 36864 data: 7 6 5 5 3 4 3 3 3 25/30 Read block 28 len 12288 26/30 Read block 489 len 4096 27/30 Write block 238 len 8192 data: 6 6 28/30 Read block 18 len 4096 29/30 Read block 453 len 16384 30/30 Read block 400 len 20480 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 5 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:18.024591608Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 105 105 105 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 60 60 60 104 104 104 T T T 3 060-079 54 54 54 89 89 89 T T T 4 080-099 60 60 60 104 104 104 F F F 5 100-119 58 58 58 102 102 102 F F F 6 120-139 60 60 60 104 104 104 T T T 7 140-159 62 62 62 105 105 105 T T T 8 160-179 60 60 60 104 104 104 F F F 9 180-199 60 60 60 104 104 104 F F F 10 200-219 59 59 59 103 103 103 F F F 11 220-239 60 60 60 104 104 104 T T T 12 240-259 60 60 60 104 104 104 F F F 13 260-279 60 60 60 104 104 104 F F F 14 280-299 60 60 60 104 104 104 F F F 15 300-319 59 59 59 103 103 103 T T T 16 320-339 60 60 60 104 104 104 T T T 17 340-359 58 58 58 101 101 101 T T T 18 360-379 56 56 56 98 98 98 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 58 58 58 101 101 101 T T T 21 420-439 58 58 58 101 101 101 F F F 22 440-459 56 56 56 97 97 97 T T T 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 54 54 54 89 89 89 T T T 27 540-559 60 60 60 104 104 104 T T T 28 560-579 50 50 50 86 86 86 T T T 29 580-599 58 58 58 100 100 100 F F F Max gen: 62, Max flush: 105 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 105 105 105 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 63 63 63 106 106 106 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 60 60 60 104 104 104 F F F 5 100-119 58 58 58 102 102 102 F F F 6 120-139 63 63 63 106 106 106 F F F 7 140-159 63 63 63 106 106 106 F F F 8 160-179 60 60 60 104 104 104 F F F 9 180-199 60 60 60 104 104 104 F F F 10 200-219 59 59 59 103 103 103 F F F 11 220-239 63 63 63 106 106 106 F F F 12 240-259 60 60 60 104 104 104 F F F 13 260-279 60 60 60 104 104 104 F F F 14 280-299 60 60 60 104 104 104 F F F 15 300-319 63 63 63 106 106 106 F F F 16 320-339 63 63 63 106 106 106 F F F 17 340-359 63 63 63 106 106 106 F F F 18 360-379 56 56 56 98 98 98 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 63 63 63 106 106 106 F F F 21 420-439 58 58 58 101 101 101 F F F 22 440-459 63 63 63 106 106 106 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 51 51 51 87 87 87 F F F 26 520-539 63 63 63 106 106 106 F F F 27 540-559 63 63 63 106 106 106 F F F 28 560-579 63 63 63 106 106 106 F F F 29 580-599 58 58 58 100 100 100 F F F Max gen: 63, Max flush: 106 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.316361001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.360515284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.361278311Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.361318125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.361333107Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.361347758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446} {"msg":"Crucible 5f2f15ee-dbcc-42da-9fb6-e7b1ea717c36 has session id: e4d16eac-7476-4758-8c1c-caece9520d65","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.361884102Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Upstairs opts: Upstairs UUID: 5f2f15ee-dbcc-42da-9fb6-e7b1ea717c36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.361924326Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Crucible stats registered with UUID: 5f2f15ee-dbcc-42da-9fb6-e7b1ea717c36","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.361945379Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:32.36195875Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362515526Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362543339Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446} {"msg":"5f2f15ee-dbcc-42da-9fb6-e7b1ea717c36 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362642699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362667871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"0","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362693924Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"0","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362709396Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362724037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362737238Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"2","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.36275128Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"2","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362764391Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362777983Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.362791644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"ds_connection connected from Ok(127.0.0.1:47765)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.363015867Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"ds_connection connected from Ok(127.0.0.1:61237)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.363066632Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:32.363104365Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:32.363142429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:32.363182213Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.363418887Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.363911337Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"2","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.363937189Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"0","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.364526109Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"2","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.364653442Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"0","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.365177005Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:32.365279035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.365135089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"ds_connection connected from Ok(127.0.0.1:47704)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.365390915Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.366319938Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.366884446Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[0]R flush_numbers[0..12]: [105, 104, 106, 106, 104, 102, 106, 106, 104, 104, 103, 106]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367486016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[0]R generation[0..12]: [62, 60, 63, 63, 60, 58, 63, 63, 60, 60, 59, 63]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367510089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.36752808Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[1]R flush_numbers[0..12]: [105, 104, 106, 106, 104, 102, 106, 106, 104, 104, 103, 106]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367545422Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[1]R generation[0..12]: [62, 60, 63, 63, 60, 58, 63, 63, 60, 60, 59, 63]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367562314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367579766Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[2]R flush_numbers[0..12]: [105, 104, 106, 106, 104, 102, 106, 106, 104, 104, 103, 106]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367596797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[2]R generation[0..12]: [62, 60, 63, 63, 60, 58, 63, 63, 60, 60, 59, 63]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:28:42.367613669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367633251Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367649453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367665434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367681076Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367700088Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"":"downstairs","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367716329Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367730921Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"0","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367747282Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"1","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367763054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"client":"2","":"downstairs","session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367779046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367793907Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"5f2f15ee-dbcc-42da-9fb6-e7b1ea717c36 is now active with session: e4d16eac-7476-4758-8c1c-caece9520d65","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.367809569Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:42.36782406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446,"session_id":"e4d16eac-7476-4758-8c1c-caece9520d65"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:28:42.367839012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 193 len 40960 data: 5 5 6 4 5 4 6 5 6 7 02/30 Write block 366 len 40960 data: 3 4 3 3 3 5 5 4 4 3 03/30 Write block 100 len 8192 data: 3 3 04/30 Write block 3 len 40960 data: 7 8 7 7 7 7 6 6 3 3 05/30 Flush 06/30 Write block 527 len 8192 data: 6 6 07/30 Write block 528 len 8192 data: 7 8 08/30 Read block 579 len 16384 09/30 Flush 10/30 Write block 10 len 32768 data: 7 4 4 5 5 8 8 7 11/30 Flush 12/30 Write block 272 len 40960 data: 2 3 3 3 4 4 4 4 3 3 13/30 Write block 563 len 36864 data: 3 3 4 4 4 3 4 4 4 14/30 Read block 150 len 4096 15/30 Read block 171 len 24576 16/30 Write block 206 len 40960 data: 6 5 4 5 6 5 5 5 5 6 17/30 Write block 519 len 4096 data: 9 18/30 Read block 178 len 24576 19/30 Read block 276 len 36864 20/30 Read block 462 len 36864 21/30 Read block 516 len 12288 22/30 Read block 587 len 16384 23/30 Read block 226 len 32768 24/30 Flush 25/30 Flush 26/30 Write block 414 len 12288 data: 10 9 7 27/30 Read block 69 len 36864 28/30 Read block 346 len 4096 29/30 Write block 235 len 12288 data: 7 9 7 30/30 Read block 365 len 8192 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 5 3 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:42.388118287Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 109 109 109 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 63 63 63 106 106 106 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 60 60 60 104 104 104 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 63 63 63 106 106 106 F F F 7 140-159 63 63 63 106 106 106 F F F 8 160-179 60 60 60 104 104 104 F F F 9 180-199 64 64 64 107 107 107 F F F 10 200-219 64 64 64 110 110 110 F F F 11 220-239 63 63 63 106 106 106 T T T 12 240-259 60 60 60 104 104 104 F F F 13 260-279 64 64 64 110 110 110 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 63 63 63 106 106 106 F F F 16 320-339 63 63 63 106 106 106 F F F 17 340-359 63 63 63 106 106 106 F F F 18 360-379 64 64 64 107 107 107 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 63 63 63 106 106 106 T T T 21 420-439 58 58 58 101 101 101 F F F 22 440-459 63 63 63 106 106 106 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 64 64 64 110 110 110 F F F 26 520-539 64 64 64 108 108 108 F F F 27 540-559 63 63 63 106 106 106 F F F 28 560-579 64 64 64 110 110 110 F F F 29 580-599 58 58 58 100 100 100 F F F Max gen: 64, Max flush: 110 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 109 109 109 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 63 63 63 106 106 106 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 60 60 60 104 104 104 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 63 63 63 106 106 106 F F F 7 140-159 63 63 63 106 106 106 F F F 8 160-179 60 60 60 104 104 104 F F F 9 180-199 64 64 64 107 107 107 F F F 10 200-219 64 64 64 110 110 110 F F F 11 220-239 65 65 65 111 111 111 F F F 12 240-259 60 60 60 104 104 104 F F F 13 260-279 64 64 64 110 110 110 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 63 63 63 106 106 106 F F F 16 320-339 63 63 63 106 106 106 F F F 17 340-359 63 63 63 106 106 106 F F F 18 360-379 64 64 64 107 107 107 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 65 65 65 111 111 111 F F F 21 420-439 58 58 58 101 101 101 F F F 22 440-459 63 63 63 106 106 106 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 64 64 64 110 110 110 F F F 26 520-539 64 64 64 108 108 108 F F F 27 540-559 63 63 63 106 106 106 F F F 28 560-579 64 64 64 110 110 110 F F F 29 580-599 58 58 58 100 100 100 F F F Max gen: 65, Max flush: 111 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.619104072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.663754765Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.664418642Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.664460686Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.664475278Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.664489419Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454} {"msg":"Crucible 61a84359-c634-4d8a-bca1-dac8d5fca30b has session id: f572fe18-df8e-4926-ad4d-a6403be48729","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665078718Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Upstairs opts: Upstairs UUID: 61a84359-c634-4d8a-bca1-dac8d5fca30b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665106591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Crucible stats registered with UUID: 61a84359-c634-4d8a-bca1-dac8d5fca30b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665121163Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:46.665133664Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665556897Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665583929Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454} {"msg":"61a84359-c634-4d8a-bca1-dac8d5fca30b active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665629914Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665650766Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"0","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665676649Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"0","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.6656908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"1","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665705012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"1","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665717953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665732144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665745136Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665764228Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665872949Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"ds_connection connected from Ok(127.0.0.1:48246)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.665933175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:46.665976539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:46.666016383Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:46.666052887Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"ds_connection connected from Ok(127.0.0.1:53981)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.666162418Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.666217433Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.666654268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"0","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.666839176Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"1","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.667312534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"0","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.667451888Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"1","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.667922575Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:46.668075581Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.666876889Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"ds_connection connected from Ok(127.0.0.1:58570)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.667074799Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.668160798Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.668737946Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[0]R flush_numbers[0..12]: [109, 104, 106, 106, 104, 107, 106, 106, 104, 107, 110, 111]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669308364Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[0]R generation[0..12]: [64, 60, 63, 63, 60, 64, 63, 63, 60, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669329676Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669344818Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[1]R flush_numbers[0..12]: [109, 104, 106, 106, 104, 107, 106, 106, 104, 107, 110, 111]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669363019Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[1]R generation[0..12]: [64, 60, 63, 63, 60, 64, 63, 63, 60, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669376631Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669390052Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[2]R flush_numbers[0..12]: [109, 104, 106, 106, 104, 107, 106, 106, 104, 107, 110, 111]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669403434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[2]R generation[0..12]: [64, 60, 63, 63, 60, 64, 63, 63, 60, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669416575Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669430306Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669442787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:28:56.669455589Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669472801Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669495673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"":"downstairs","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669520565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669536387Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"0","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669549908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"1","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.66956669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"client":"2","":"downstairs","session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669591002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669616135Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"61a84359-c634-4d8a-bca1-dac8d5fca30b is now active with session: f572fe18-df8e-4926-ad4d-a6403be48729","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669638767Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669659349Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454,"session_id":"f572fe18-df8e-4926-ad4d-a6403be48729"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:28:56.669683572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 232 len 16384 data: 6 6 5 8 04/30 Write block 584 len 40960 data: 4 4 4 4 4 4 6 7 6 7 05/30 Write block 350 len 32768 data: 5 4 4 4 3 3 4 5 06/30 Flush 07/30 Write block 251 len 28672 data: 10 11 12 9 6 4 5 08/30 Read block 584 len 36864 09/30 Read block 517 len 24576 10/30 Read block 36 len 24576 11/30 Read block 108 len 32768 12/30 Read block 87 len 4096 13/30 Write block 302 len 4096 data: 10 14/30 Read block 73 len 16384 15/30 Read block 16 len 4096 16/30 Read block 589 len 12288 17/30 Read block 35 len 4096 18/30 Read block 413 len 36864 19/30 Write block 6 len 12288 data: 8 8 8 20/30 Read block 280 len 4096 21/30 Write block 408 len 36864 data: 9 9 10 7 11 12 11 10 8 22/30 Read block 194 len 8192 23/30 Write block 300 len 16384 data: 9 8 11 10 24/30 Write block 237 len 36864 data: 8 7 7 4 4 3 4 5 5 25/30 Write block 195 len 24576 data: 7 5 6 5 7 6 26/30 Read block 377 len 16384 27/30 Write block 179 len 16384 data: 4 4 4 4 28/30 Write block 408 len 20480 data: 10 10 11 8 12 29/30 Write block 576 len 8192 data: 5 5 30/30 Write block 6 len 40960 data: 9 9 9 7 8 5 5 6 6 9 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 7 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 5 Sent Done Sent false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 1 3 7 Done 21 23 21 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:28:56.688874637Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 109 109 109 T T T 1 020-039 60 60 60 104 104 104 F F F 2 040-059 63 63 63 106 106 106 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 60 60 60 104 104 104 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 63 63 63 106 106 106 F F F 7 140-159 63 63 63 106 106 106 F F F 8 160-179 60 60 60 104 104 104 T T T 9 180-199 64 64 64 107 107 107 T T T 10 200-219 64 64 64 110 110 110 T T T 11 220-239 66 66 66 114 114 114 T T T 12 240-259 60 60 60 104 104 104 T T T 13 260-279 64 64 64 110 110 110 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 63 63 63 106 106 106 T T T 16 320-339 63 63 63 106 106 106 F F F 17 340-359 66 66 66 114 114 114 F F F 18 360-379 64 64 64 107 107 107 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 65 65 65 111 111 111 T T T 21 420-439 58 58 58 101 101 101 F F F 22 440-459 63 63 63 106 106 106 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 64 64 64 110 110 110 F F F 26 520-539 64 64 64 108 108 108 F F F 27 540-559 63 63 63 106 106 106 F F F 28 560-579 64 64 64 110 110 110 T T T 29 580-599 66 66 66 114 114 114 F F F Max gen: 66, Max flush: 114 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 115 115 115 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 63 63 63 106 106 106 F F F 3 060-079 63 63 63 106 106 106 F F F 4 080-099 60 60 60 104 104 104 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 63 63 63 106 106 106 F F F 7 140-159 63 63 63 106 106 106 F F F 8 160-179 67 67 67 115 115 115 F F F 9 180-199 67 67 67 115 115 115 F F F 10 200-219 67 67 67 115 115 115 F F F 11 220-239 67 67 67 115 115 115 F F F 12 240-259 67 67 67 115 115 115 F F F 13 260-279 64 64 64 110 110 110 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 67 67 67 115 115 115 F F F 16 320-339 63 63 63 106 106 106 F F F 17 340-359 66 66 66 114 114 114 F F F 18 360-379 64 64 64 107 107 107 F F F 19 380-399 58 58 58 102 102 102 F F F 20 400-419 67 67 67 115 115 115 F F F 21 420-439 58 58 58 101 101 101 F F F 22 440-459 63 63 63 106 106 106 F F F 23 460-479 58 58 58 101 101 101 F F F 24 480-499 58 58 58 101 101 101 F F F 25 500-519 64 64 64 110 110 110 F F F 26 520-539 64 64 64 108 108 108 F F F 27 540-559 63 63 63 106 106 106 F F F 28 560-579 67 67 67 115 115 115 F F F 29 580-599 66 66 66 114 114 114 F F F Max gen: 67, Max flush: 115 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:10.958540622Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.002552901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.003278194Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.003320609Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.00333559Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.003350171Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462} {"msg":"Crucible 3adc80b2-3746-4f02-aef6-722b6c516335 has session id: fa7a3375-e61e-4496-b814-fc648ca38045","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.003926219Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Upstairs opts: Upstairs UUID: 3adc80b2-3746-4f02-aef6-722b6c516335, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.003961203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Crucible stats registered with UUID: 3adc80b2-3746-4f02-aef6-722b6c516335","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.003977075Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:11.003989216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004441242Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004465004Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462} {"msg":"3adc80b2-3746-4f02-aef6-722b6c516335 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.00452454Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004546262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"0","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004574595Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"0","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004589456Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004603958Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004618479Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"2","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004643772Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"2","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004669565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.004697377Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.0047251Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:11.00492245Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"ds_connection connected from Ok(127.0.0.1:45564)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.005064514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"ds_connection connected from Ok(127.0.0.1:37765)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.005092047Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:11.00511918Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:11.005146033Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.005169995Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.00561886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"2","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.005655864Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"0","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.006259985Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"0","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.006288178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"2","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.006920232Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:11.006969326Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.005664385Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"ds_connection connected from Ok(127.0.0.1:42695)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.005795938Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.00700113Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.007593819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[0]R flush_numbers[0..12]: [115, 104, 106, 106, 104, 107, 106, 106, 115, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008210592Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[0]R generation[0..12]: [67, 60, 63, 63, 60, 64, 63, 63, 67, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008231784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008247555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[1]R flush_numbers[0..12]: [115, 104, 106, 106, 104, 107, 106, 106, 115, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008261847Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[1]R generation[0..12]: [67, 60, 63, 63, 60, 64, 63, 63, 67, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008280229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.00829442Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[2]R flush_numbers[0..12]: [115, 104, 106, 106, 104, 107, 106, 106, 115, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008308131Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[2]R generation[0..12]: [67, 60, 63, 63, 60, 64, 63, 63, 67, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008322803Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008336534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008349246Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008362277Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008380279Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.00839342Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"":"downstairs","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008406271Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008417893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"0","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008430614Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"1","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008443375Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"client":"2","":"downstairs","session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008456186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008470368Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"3adc80b2-3746-4f02-aef6-722b6c516335 is now active with session: fa7a3375-e61e-4496-b814-fc648ca38045","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008483019Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.00849466Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462,"session_id":"fa7a3375-e61e-4496-b814-fc648ca38045"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:21.008506482Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 436 len 28672 data: 5 5 7 10 8 8 7 02/30 Read block 489 len 32768 03/30 Flush 04/30 Write block 308 len 8192 data: 4 5 05/30 Read block 199 len 4096 06/30 Read block 514 len 28672 07/30 Read block 432 len 20480 08/30 Write block 159 len 28672 data: 4 3 4 4 5 5 5 09/30 Flush 10/30 Write block 404 len 8192 data: 4 4 11/30 Write block 394 len 40960 data: 8 9 9 8 4 3 4 4 4 3 12/30 Write block 418 len 12288 data: 7 6 5 13/30 Read block 587 len 20480 14/30 Read block 520 len 8192 15/30 Write block 70 len 36864 data: 4 5 4 5 5 7 7 5 6 16/30 Read block 200 len 4096 17/30 Flush 18/30 Write block 134 len 40960 data: 9 9 10 9 8 10 12 11 10 9 19/30 Write block 157 len 8192 data: 6 6 20/30 Flush 21/30 Write block 540 len 40960 data: 5 7 8 8 7 8 8 9 9 9 22/30 Write block 482 len 4096 data: 4 23/30 Read block 429 len 20480 24/30 Write block 482 len 4096 data: 5 25/30 Write block 516 len 20480 data: 6 6 7 10 8 26/30 Write block 471 len 24576 data: 10 8 5 3 3 3 27/30 Write block 369 len 36864 data: 4 4 6 6 5 5 4 3 4 28/30 Write block 326 len 4096 data: 6 29/30 Read block 353 len 28672 30/30 Write block 138 len 12288 data: 9 11 13 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:21.028114129Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 115 115 115 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 63 63 63 106 106 106 F F F 3 060-079 68 68 68 118 118 118 F F F 4 080-099 60 60 60 104 104 104 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 68 68 68 119 119 119 T T T 7 140-159 68 68 68 119 119 119 T T T 8 160-179 68 68 68 117 117 117 F F F 9 180-199 67 67 67 115 115 115 F F F 10 200-219 67 67 67 115 115 115 F F F 11 220-239 67 67 67 115 115 115 F F F 12 240-259 67 67 67 115 115 115 F F F 13 260-279 64 64 64 110 110 110 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 68 68 68 117 117 117 F F F 16 320-339 63 63 63 106 106 106 T T T 17 340-359 66 66 66 114 114 114 F F F 18 360-379 64 64 64 107 107 107 T T T 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 58 58 58 101 101 101 T T T 24 480-499 58 58 58 101 101 101 T T T 25 500-519 64 64 64 110 110 110 T T T 26 520-539 64 64 64 108 108 108 T T T 27 540-559 63 63 63 106 106 106 T T T 28 560-579 67 67 67 115 115 115 F F F 29 580-599 66 66 66 114 114 114 F F F Max gen: 68, Max flush: 119 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 115 115 115 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 63 63 63 106 106 106 F F F 3 060-079 68 68 68 118 118 118 F F F 4 080-099 60 60 60 104 104 104 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 69 69 69 120 120 120 F F F 7 140-159 69 69 69 120 120 120 F F F 8 160-179 68 68 68 117 117 117 F F F 9 180-199 67 67 67 115 115 115 F F F 10 200-219 67 67 67 115 115 115 F F F 11 220-239 67 67 67 115 115 115 F F F 12 240-259 67 67 67 115 115 115 F F F 13 260-279 64 64 64 110 110 110 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 68 68 68 117 117 117 F F F 16 320-339 69 69 69 120 120 120 F F F 17 340-359 66 66 66 114 114 114 F F F 18 360-379 69 69 69 120 120 120 F F F 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 69 69 69 120 120 120 F F F 25 500-519 69 69 69 120 120 120 F F F 26 520-539 69 69 69 120 120 120 F F F 27 540-559 69 69 69 120 120 120 F F F 28 560-579 67 67 67 115 115 115 F F F 29 580-599 66 66 66 114 114 114 F F F Max gen: 69, Max flush: 120 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.292654498Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.336866077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.337643985Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.33768873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.337703631Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.337718033Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470} {"msg":"Crucible d45871c0-5e6a-41f2-af20-117a1810e51a has session id: 6876a2d1-9849-4130-8a6b-e800cb8ddb56","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.338346656Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Upstairs opts: Upstairs UUID: d45871c0-5e6a-41f2-af20-117a1810e51a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.33838691Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Crucible stats registered with UUID: d45871c0-5e6a-41f2-af20-117a1810e51a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.338403502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:35.338423514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.338982891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339017634Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470} {"msg":"d45871c0-5e6a-41f2-af20-117a1810e51a active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.33907744Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339109873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"0","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339147427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"0","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339185951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"1","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339217124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"1","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339256218Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"2","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339292732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"2","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339327685Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339359398Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.33937806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"ds_connection connected from Ok(127.0.0.1:47179)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339565649Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"ds_connection connected from Ok(127.0.0.1:57226)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339607514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"ds_connection connected from Ok(127.0.0.1:50292)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.339638957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.340250618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"0","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.340436167Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"1","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.3408625Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"0","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.341000284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"1","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.341473952Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.341592434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.341858481Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"2","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342362821Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"2","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"[0]R flush_numbers[0..12]: [115, 104, 106, 118, 104, 107, 120, 120, 117, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342875033Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"[0]R generation[0..12]: [67, 60, 63, 68, 60, 64, 69, 69, 68, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342897405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342912037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"[1]R flush_numbers[0..12]: [115, 104, 106, 118, 104, 107, 120, 120, 117, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342926048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"[1]R generation[0..12]: [67, 60, 63, 68, 60, 64, 69, 69, 68, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342939979Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342953831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"[2]R flush_numbers[0..12]: [115, 104, 106, 118, 104, 107, 120, 120, 117, 115, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342967112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"[2]R generation[0..12]: [67, 60, 63, 68, 60, 64, 69, 69, 68, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342980363Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.342999836Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343012727Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343025508Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.3430381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343054951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"":"downstairs","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343068132Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343079764Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"0","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343092575Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"1","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343105116Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"client":"2","":"downstairs","session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343117848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343129719Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"d45871c0-5e6a-41f2-af20-117a1810e51a is now active with session: 6876a2d1-9849-4130-8a6b-e800cb8ddb56","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.3431423Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343158192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470,"session_id":"6876a2d1-9849-4130-8a6b-e800cb8ddb56"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:35.343170063Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 264 len 20480 data: 4 3 3 5 5 03/30 Write block 253 len 40960 data: 13 10 7 5 6 5 6 5 6 4 04/30 Read block 19 len 32768 05/30 Read block 451 len 12288 06/30 Read block 257 len 28672 07/30 Flush 08/30 Write block 141 len 28672 data: 12 11 10 6 5 4 5 09/30 Flush 10/30 Flush 11/30 Write block 252 len 12288 data: 12 14 11 12/30 Read block 459 len 4096 13/30 Write block 52 len 40960 data: 5 5 7 8 7 8 7 6 6 7 14/30 Read block 409 len 8192 15/30 Write block 256 len 4096 data: 6 16/30 Read block 401 len 28672 17/30 Write block 521 len 32768 data: 8 9 7 7 6 6 7 8 18/30 Flush 19/30 Write block 572 len 8192 data: 5 6 20/30 Write block 217 len 20480 data: 5 5 6 5 5 21/30 Read block 18 len 12288 22/30 Flush 23/30 Write block 360 len 40960 data: 3 3 3 4 4 4 4 5 4 5 24/30 Read block 549 len 4096 25/30 Write block 211 len 32768 data: 6 6 6 6 7 6 6 6 26/30 Write block 525 len 32768 data: 7 7 8 9 9 8 7 5 27/30 Write block 151 len 40960 data: 8 7 7 7 7 5 7 7 5 4 28/30 Write block 79 len 24576 data: 7 7 6 4 6 6 29/30 Read block 112 len 40960 30/30 Read block 393 len 28672 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:35.36356851Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 115 115 115 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 70 70 70 125 125 125 F F F 3 060-079 70 70 70 125 125 125 T T T 4 080-099 60 60 60 104 104 104 T T T 5 100-119 64 64 64 107 107 107 F F F 6 120-139 69 69 69 120 120 120 F F F 7 140-159 70 70 70 123 123 123 T T T 8 160-179 68 68 68 117 117 117 T T T 9 180-199 67 67 67 115 115 115 F F F 10 200-219 70 70 70 126 126 126 T T T 11 220-239 70 70 70 126 126 126 F F F 12 240-259 70 70 70 125 125 125 F F F 13 260-279 70 70 70 122 122 122 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 68 68 68 117 117 117 F F F 16 320-339 69 69 69 120 120 120 F F F 17 340-359 66 66 66 114 114 114 F F F 18 360-379 69 69 69 120 120 120 T T T 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 69 69 69 120 120 120 F F F 25 500-519 69 69 69 120 120 120 F F F 26 520-539 70 70 70 125 125 125 T T T 27 540-559 69 69 69 120 120 120 F F F 28 560-579 70 70 70 126 126 126 F F F 29 580-599 66 66 66 114 114 114 F F F Max gen: 70, Max flush: 126 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 115 115 115 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 70 70 70 125 125 125 F F F 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 69 69 69 120 120 120 F F F 7 140-159 71 71 71 127 127 127 F F F 8 160-179 71 71 71 127 127 127 F F F 9 180-199 67 67 67 115 115 115 F F F 10 200-219 71 71 71 127 127 127 F F F 11 220-239 70 70 70 126 126 126 F F F 12 240-259 70 70 70 125 125 125 F F F 13 260-279 70 70 70 122 122 122 F F F 14 280-299 64 64 64 110 110 110 F F F 15 300-319 68 68 68 117 117 117 F F F 16 320-339 69 69 69 120 120 120 F F F 17 340-359 66 66 66 114 114 114 F F F 18 360-379 71 71 71 127 127 127 F F F 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 68 68 68 118 118 118 F F F 22 440-459 68 68 68 116 116 116 F F F 23 460-479 69 69 69 120 120 120 F F F 24 480-499 69 69 69 120 120 120 F F F 25 500-519 69 69 69 120 120 120 F F F 26 520-539 71 71 71 127 127 127 F F F 27 540-559 69 69 69 120 120 120 F F F 28 560-579 70 70 70 126 126 126 F F F 29 580-599 66 66 66 114 114 114 F F F Max gen: 71, Max flush: 127 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.616230371Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:39.659323507Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.65974991Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.691118274Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.69177573Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.691809754Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.691824465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.691839287Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478} {"msg":"Crucible 1d44b582-4ba2-41f8-9b6a-2c0bc5d160b3 has session id: 09baa70c-5ab5-4a94-9191-136cc79cca24","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.692523216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Upstairs opts: Upstairs UUID: 1d44b582-4ba2-41f8-9b6a-2c0bc5d160b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.692574591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Crucible stats registered with UUID: 1d44b582-4ba2-41f8-9b6a-2c0bc5d160b3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.692596583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:39.692613775Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693028787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.69306029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478} {"msg":"1d44b582-4ba2-41f8-9b6a-2c0bc5d160b3 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693119366Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693148069Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"0","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693184912Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"0","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693219206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"1","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693248569Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"1","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693275161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"2","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693298374Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"2","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693326027Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.69335745Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693377782Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"ds_connection connected from Ok(127.0.0.1:48145)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693478732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"ds_connection connected from Ok(127.0.0.1:43041)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693531787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"ds_connection connected from Ok(127.0.0.1:42065)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.693570231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.69425432Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"1","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.694515317Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"0","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.694544849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"2","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.694904776Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"1","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.695223378Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"0","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.695259262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"2","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.695555371Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.695815688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[0]R flush_numbers[0..12]: [115, 104, 125, 127, 127, 107, 120, 127, 127, 115, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.69593745Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[0]R generation[0..12]: [67, 60, 70, 71, 71, 64, 69, 71, 71, 67, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.695956122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.695970823Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[1]R flush_numbers[0..12]: [115, 104, 125, 127, 127, 107, 120, 127, 127, 115, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.695984465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[1]R generation[0..12]: [67, 60, 70, 71, 71, 64, 69, 71, 71, 67, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696012068Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696027089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[2]R flush_numbers[0..12]: [115, 104, 125, 127, 127, 107, 120, 127, 127, 115, 127, 126]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696052001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[2]R generation[0..12]: [67, 60, 70, 71, 71, 64, 69, 71, 71, 67, 71, 70]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696066573Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696080184Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696092936Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"Wait for a query_work_queue command to finish before sending IO msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696114598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696127329Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.69613971Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"":"downstairs","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696152552Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696164183Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"0","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696176884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"1","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696189555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"client":"2","":"downstairs","session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696202327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696214058Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"1d44b582-4ba2-41f8-9b6a-2c0bc5d160b3 is now active with session: 09baa70c-5ab5-4a94-9191-136cc79cca24","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696226489Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.69623815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478,"session_id":"09baa70c-5ab5-4a94-9191-136cc79cca24"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:39.696251752Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 192 len 16384 data: 4 6 6 8 02/30 Write block 588 len 36864 data: 5 5 7 8 7 8 6 3 2 03/30 Read block 461 len 12288 04/30 Write block 8 len 36864 data: 10 8 9 6 6 7 7 10 9 05/30 Read block 522 len 28672 06/30 Write block 341 len 8192 data: 5 6 07/30 Read block 225 len 36864 08/30 Read block 82 len 4096 09/30 Write block 223 len 24576 data: 7 8 10 9 7 7 10/30 Read block 159 len 4096 11/30 Write block 185 len 8192 data: 7 6 12/30 Write block 272 len 4096 data: 3 13/30 Write block 324 len 4096 data: 8 14/30 Write block 572 len 16384 data: 6 7 6 7 15/30 Read block 282 len 20480 16/30 Write block 181 len 36864 data: 5 5 4 7 8 7 5 4 3 17/30 Write block 457 len 40960 data: 6 7 6 5 8 7 6 9 10 10 18/30 Read block 123 len 4096 19/30 Write block 141 len 4096 data: 13 20/30 Write block 299 len 36864 data: 7 10 9 12 11 11 7 7 5 21/30 Read block 172 len 32768 22/30 Read block 171 len 28672 23/30 Read block 447 len 16384 24/30 Flush 25/30 Write block 233 len 24576 data: 7 6 9 10 9 8 26/30 Read block 65 len 32768 27/30 Read block 12 len 16384 28/30 Write block 422 len 4096 data: 6 29/30 Write block 589 len 12288 data: 6 8 9 30/30 Write block 132 len 8192 data: 7 8 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:39.71467367Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 128 128 128 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 70 70 70 125 125 125 F F F 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 69 69 69 120 120 120 T T T 7 140-159 72 72 72 128 128 128 F F F 8 160-179 71 71 71 127 127 127 F F F 9 180-199 72 72 72 128 128 128 F F F 10 200-219 71 71 71 127 127 127 F F F 11 220-239 72 72 72 128 128 128 T T T 12 240-259 70 70 70 125 125 125 F F F 13 260-279 72 72 72 128 128 128 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 72 72 72 128 128 128 F F F 16 320-339 72 72 72 128 128 128 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 71 71 71 127 127 127 F F F 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 68 68 68 118 118 118 T T T 22 440-459 72 72 72 128 128 128 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 69 69 69 120 120 120 F F F 25 500-519 69 69 69 120 120 120 F F F 26 520-539 71 71 71 127 127 127 F F F 27 540-559 69 69 69 120 120 120 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 72 72 72 128 128 128 T T T Max gen: 72, Max flush: 128 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 128 128 128 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 70 70 70 125 125 125 F F F 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 64 64 64 107 107 107 F F F 6 120-139 73 73 73 129 129 129 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 71 71 71 127 127 127 F F F 9 180-199 72 72 72 128 128 128 F F F 10 200-219 71 71 71 127 127 127 F F F 11 220-239 73 73 73 129 129 129 F F F 12 240-259 70 70 70 125 125 125 F F F 13 260-279 72 72 72 128 128 128 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 72 72 72 128 128 128 F F F 16 320-339 72 72 72 128 128 128 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 71 71 71 127 127 127 F F F 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 73 73 73 129 129 129 F F F 22 440-459 72 72 72 128 128 128 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 69 69 69 120 120 120 F F F 25 500-519 69 69 69 120 120 120 F F F 26 520-539 71 71 71 127 127 127 F F F 27 540-559 69 69 69 120 120 120 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 73, Max flush: 129 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:43.978133689Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.022195813Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.022919636Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.02296286Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.022977882Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.022996274Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486} {"msg":"Crucible 911990b0-ceba-4d82-aa3a-6cf824060234 has session id: ec7640fa-c180-4620-8cff-7cf92d20ff88","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.023574862Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Upstairs opts: Upstairs UUID: 911990b0-ceba-4d82-aa3a-6cf824060234, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.023611106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Crucible stats registered with UUID: 911990b0-ceba-4d82-aa3a-6cf824060234","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.023634298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:44.0236564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024182473Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024208426Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486} {"msg":"911990b0-ceba-4d82-aa3a-6cf824060234 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024270272Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024288014Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"0","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024311226Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"0","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024326908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"1","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.02434125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"1","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024354571Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024368302Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024381394Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024397925Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024431748Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"ds_connection connected from Ok(127.0.0.1:34305)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024580583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"ds_connection connected from Ok(127.0.0.1:50987)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024617257Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:44.02464472Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:44.024672673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:44.024709927Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.024834199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.025278204Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"0","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.025306987Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"1","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.02593231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"0","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.025958973Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"1","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.026594067Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:44.026704898Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.025727369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"ds_connection connected from Ok(127.0.0.1:49655)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.025877444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.026944122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.027493698Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[0]R flush_numbers[0..12]: [128, 104, 125, 127, 127, 107, 129, 128, 127, 128, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028063275Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[0]R generation[0..12]: [72, 60, 70, 71, 71, 64, 73, 72, 71, 72, 71, 73]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028082957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028097668Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[1]R flush_numbers[0..12]: [128, 104, 125, 127, 127, 107, 129, 128, 127, 128, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.02811215Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[1]R generation[0..12]: [72, 60, 70, 71, 71, 64, 73, 72, 71, 72, 71, 73]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028126021Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028140013Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[2]R flush_numbers[0..12]: [128, 104, 125, 127, 127, 107, 129, 128, 127, 128, 127, 129]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028153574Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[2]R generation[0..12]: [72, 60, 70, 71, 71, 64, 73, 72, 71, 72, 71, 73]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028166746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028180457Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028197029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.0282101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028222571Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"All extents match","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-02-27T17:29:54.028241253Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"":"downstairs","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028263405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028285587Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"0","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.02830942Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"1","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028333982Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"client":"2","":"downstairs","session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028358515Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028380957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"911990b0-ceba-4d82-aa3a-6cf824060234 is now active with session: ec7640fa-c180-4620-8cff-7cf92d20ff88","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028404079Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:29:54.028424712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486,"session_id":"ec7640fa-c180-4620-8cff-7cf92d20ff88"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:29:54.028446744Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 5 len 4096 data: 8 03/30 Write block 118 len 40960 data: 12 11 13 12 10 10 10 9 7 7 04/30 Flush 05/30 Write block 378 len 8192 data: 4 5 06/30 Read block 561 len 12288 07/30 Read block 395 len 36864 08/30 Write block 509 len 24576 data: 6 5 4 5 6 6 09/30 Read block 262 len 4096 10/30 Write block 334 len 16384 data: 9 9 10 7 11/30 Write block 13 len 8192 data: 8 8 12/30 Write block 253 len 32768 data: 15 12 8 7 7 6 7 6 13/30 Read block 578 len 24576 14/30 Read block 61 len 28672 15/30 Write block 332 len 20480 data: 9 8 10 10 11 16/30 Write block 108 len 12288 data: 7 9 10 17/30 Read block 361 len 24576 18/30 Write block 259 len 32768 data: 8 7 7 5 4 5 4 4 19/30 Read block 25 len 32768 20/30 Read block 20 len 16384 21/30 Read block 66 len 8192 22/30 Read block 157 len 28672 23/30 Read block 0 len 20480 24/30 Read block 182 len 32768 25/30 Read block 252 len 28672 26/30 Write block 216 len 40960 data: 7 7 7 7 6 6 8 8 9 11 27/30 Read block 92 len 4096 28/30 Write block 512 len 20480 data: 6 7 7 6 7 29/30 Read block 259 len 4096 30/30 Write block 193 len 20480 data: 7 7 9 6 7 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:29:54.048320428Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 131 131 131 T T T 1 020-039 60 60 60 104 104 104 F F F 2 040-059 70 70 70 125 125 125 F F F 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 74 74 74 131 131 131 T T T 6 120-139 74 74 74 131 131 131 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 71 71 71 127 127 127 F F F 9 180-199 72 72 72 128 128 128 T T T 10 200-219 71 71 71 127 127 127 T T T 11 220-239 73 73 73 129 129 129 T T T 12 240-259 70 70 70 125 125 125 T T T 13 260-279 72 72 72 128 128 128 T T T 14 280-299 72 72 72 128 128 128 F F F 15 300-319 72 72 72 128 128 128 F F F 16 320-339 72 72 72 128 128 128 T T T 17 340-359 72 72 72 128 128 128 F F F 18 360-379 71 71 71 127 127 127 T T T 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 73 73 73 129 129 129 F F F 22 440-459 72 72 72 128 128 128 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 69 69 69 120 120 120 F F F 25 500-519 69 69 69 120 120 120 T T T 26 520-539 71 71 71 127 127 127 F F F 27 540-559 69 69 69 120 120 120 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 74, Max flush: 131 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 F F F 1 020-039 60 60 60 104 104 104 F F F 2 040-059 70 70 70 125 125 125 F F F 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 75 75 75 132 132 132 F F F 6 120-139 74 74 74 131 131 131 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 71 71 71 127 127 127 F F F 9 180-199 75 75 75 132 132 132 F F F 10 200-219 75 75 75 132 132 132 F F F 11 220-239 75 75 75 132 132 132 F F F 12 240-259 75 75 75 132 132 132 F F F 13 260-279 75 75 75 132 132 132 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 72 72 72 128 128 128 F F F 16 320-339 75 75 75 132 132 132 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 75 75 75 132 132 132 F F F 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 73 73 73 129 129 129 F F F 22 440-459 72 72 72 128 128 128 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 69 69 69 120 120 120 F F F 25 500-519 75 75 75 132 132 132 F F F 26 520-539 71 71 71 127 127 127 F F F 27 540-559 69 69 69 120 120 120 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 75, Max flush: 132 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.32479536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:08.367800338Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.368199748Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.399601995Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.400264442Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.400299685Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.400318278Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.400332649Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494} {"msg":"Crucible 8ae72692-7fe1-4b5a-8c91-759d6c163386 has session id: 76fb7531-d458-43b0-8628-6dbd5e32aef6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.400894276Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Upstairs opts: Upstairs UUID: 8ae72692-7fe1-4b5a-8c91-759d6c163386, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401018528Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Crucible stats registered with UUID: 8ae72692-7fe1-4b5a-8c91-759d6c163386","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401049951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:08.401072683Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401497196Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.40153263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494} {"msg":"8ae72692-7fe1-4b5a-8c91-759d6c163386 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401589066Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401620369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"0","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401650212Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"0","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401667354Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"1","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401681595Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"1","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401694836Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"2","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401714178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"2","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.40172762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401741171Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.401754542Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"ds_connection connected from Ok(127.0.0.1:40769)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.402108938Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47750)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.402160143Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35744)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.402189726Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.402843262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"1","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.402885646Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"2","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.402973665Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"0","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.403501679Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"2","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.403623511Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"1","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.403653314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"0","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404071976Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404283958Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[0]R flush_numbers[0..12]: [132, 104, 125, 127, 127, 132, 131, 128, 127, 132, 132, 132]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404359235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[0]R generation[0..12]: [75, 60, 70, 71, 71, 75, 74, 72, 71, 75, 75, 75]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404385768Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404400519Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[1]R flush_numbers[0..12]: [132, 104, 125, 127, 127, 132, 131, 128, 127, 132, 132, 132]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404415361Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[1]R generation[0..12]: [75, 60, 70, 71, 71, 75, 74, 72, 71, 75, 75, 75]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404429202Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404442874Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[2]R flush_numbers[0..12]: [132, 104, 125, 127, 127, 132, 131, 128, 127, 132, 132, 132]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404456745Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[2]R generation[0..12]: [75, 60, 70, 71, 71, 75, 74, 72, 71, 75, 75, 75]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404470036Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404484038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404496559Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.40450929Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404521991Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"All extents match","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2025-02-27T17:30:08.404546924Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"":"downstairs","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404561695Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404576817Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"0","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404590108Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"1","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404602819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"client":"2","":"downstairs","session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404615591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404627542Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"8ae72692-7fe1-4b5a-8c91-759d6c163386 is now active with session: 76fb7531-d458-43b0-8628-6dbd5e32aef6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404640263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404652044Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494,"session_id":"76fb7531-d458-43b0-8628-6dbd5e32aef6"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:08.404663916Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 309 len 32768 data: 6 6 6 6 5 4 5 5 02/30 Write block 168 len 12288 data: 4 4 5 03/30 Read block 111 len 4096 04/30 Read block 585 len 40960 05/30 Read block 485 len 16384 06/30 Read block 375 len 12288 07/30 Read block 407 len 32768 08/30 Read block 462 len 16384 09/30 Read block 192 len 8192 10/30 Write block 197 len 28672 data: 8 6 8 7 7 8 5 11/30 Read block 119 len 28672 12/30 Write block 428 len 16384 data: 6 5 5 4 13/30 Read block 85 len 32768 14/30 Read block 497 len 16384 15/30 Read block 452 len 24576 16/30 Read block 219 len 12288 17/30 Write block 425 len 8192 data: 6 6 18/30 Write block 529 len 4096 data: 10 19/30 Write block 361 len 4096 data: 4 20/30 Read block 93 len 40960 21/30 Write block 496 len 40960 data: 4 4 5 6 5 5 4 4 3 4 22/30 Read block 550 len 20480 23/30 Read block 211 len 40960 24/30 Write block 564 len 36864 data: 4 5 5 5 4 5 5 5 7 25/30 Flush 26/30 Write block 161 len 28672 data: 5 5 6 6 6 4 4 27/30 Read block 141 len 4096 28/30 Read block 428 len 4096 29/30 Read block 417 len 12288 30/30 Write block 17 len 20480 data: 8 7 8 8 7 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:08.425686946Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 T T T 1 020-039 60 60 60 104 104 104 T T T 2 040-059 70 70 70 125 125 125 F F F 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 75 75 75 132 132 132 F F F 6 120-139 74 74 74 131 131 131 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 76 76 76 133 133 133 T T T 9 180-199 76 76 76 133 133 133 F F F 10 200-219 76 76 76 133 133 133 F F F 11 220-239 75 75 75 132 132 132 F F F 12 240-259 75 75 75 132 132 132 F F F 13 260-279 75 75 75 132 132 132 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 76 76 76 133 133 133 F F F 16 320-339 75 75 75 132 132 132 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 76 76 76 133 133 133 F F F 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 76 76 76 133 133 133 F F F 22 440-459 72 72 72 128 128 128 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 76 76 76 133 133 133 F F F 27 540-559 69 69 69 120 120 120 F F F 28 560-579 76 76 76 133 133 133 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 76, Max flush: 133 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 134 134 134 F F F 1 020-039 77 77 77 134 134 134 F F F 2 040-059 70 70 70 125 125 125 F F F 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 75 75 75 132 132 132 F F F 6 120-139 74 74 74 131 131 131 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 77 77 77 134 134 134 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 76 76 76 133 133 133 F F F 11 220-239 75 75 75 132 132 132 F F F 12 240-259 75 75 75 132 132 132 F F F 13 260-279 75 75 75 132 132 132 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 76 76 76 133 133 133 F F F 16 320-339 75 75 75 132 132 132 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 76 76 76 133 133 133 F F F 19 380-399 68 68 68 118 118 118 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 76 76 76 133 133 133 F F F 22 440-459 72 72 72 128 128 128 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 76 76 76 133 133 133 F F F 27 540-559 69 69 69 120 120 120 F F F 28 560-579 76 76 76 133 133 133 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 77, Max flush: 134 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.690903833Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.73559419Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.736398562Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.736441886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.736474579Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.736489871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502} {"msg":"Crucible 143409b0-ff01-4263-8c04-f29801ed0752 has session id: eff12f40-85df-48b4-87c1-be4198ef95cc","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737046127Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Upstairs opts: Upstairs UUID: 143409b0-ff01-4263-8c04-f29801ed0752, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737087491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Crucible stats registered with UUID: 143409b0-ff01-4263-8c04-f29801ed0752","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737110993Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:12.737134556Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737553268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737587442Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502} {"msg":"143409b0-ff01-4263-8c04-f29801ed0752 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.73766761Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737695572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"0","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737730426Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"0","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737757588Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"1","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737784641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"1","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737810074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"2","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737838487Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"2","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737866439Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737892772Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737920205Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:50191)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.737947368Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:53960)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.738028616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:63531)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.738061859Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.738571191Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"1","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.738703404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"0","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.739173711Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"1","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.73935328Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"0","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.739867251Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.739946989Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.740227598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"2","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.740736139Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"2","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[0]R flush_numbers[0..12]: [134, 134, 125, 127, 127, 132, 131, 128, 134, 133, 133, 132]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741254591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[0]R generation[0..12]: [77, 77, 70, 71, 71, 75, 74, 72, 77, 76, 76, 75]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741273953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741288505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[1]R flush_numbers[0..12]: [134, 134, 125, 127, 127, 132, 131, 128, 134, 133, 133, 132]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741302126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[1]R generation[0..12]: [77, 77, 70, 71, 71, 75, 74, 72, 77, 76, 76, 75]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741315558Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741329199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[2]R flush_numbers[0..12]: [134, 134, 125, 127, 127, 132, 131, 128, 134, 133, 133, 132]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.74134281Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[2]R generation[0..12]: [77, 77, 70, 71, 71, 75, 74, 72, 77, 76, 76, 75]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741356091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741369613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741386374Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741412037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.74143569Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741457862Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"":"downstairs","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741481544Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:30:12.741502576Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"0","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741523428Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"1","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741546241Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"client":"2","":"downstairs","session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741565333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741577644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"143409b0-ff01-4263-8c04-f29801ed0752 is now active with session: eff12f40-85df-48b4-87c1-be4198ef95cc","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741590285Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741601946Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502,"session_id":"eff12f40-85df-48b4-87c1-be4198ef95cc"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:12.741613628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 213 len 28672 data: 7 7 8 8 8 8 8 02/30 Write block 216 len 40960 data: 9 9 9 9 7 7 9 9 10 12 03/30 Flush 04/30 Write block 217 len 32768 data: 10 10 10 8 8 10 10 11 05/30 Read block 306 len 24576 06/30 Write block 376 len 36864 data: 4 5 5 6 5 4 5 7 7 07/30 Read block 562 len 24576 08/30 Write block 565 len 32768 data: 6 6 6 5 6 6 6 8 09/30 Write block 305 len 28672 data: 8 8 6 5 7 7 7 10/30 Read block 156 len 32768 11/30 Read block 207 len 20480 12/30 Write block 551 len 32768 data: 8 7 7 4 4 5 6 6 13/30 Read block 351 len 32768 14/30 Read block 25 len 8192 15/30 Read block 578 len 32768 16/30 Flush 17/30 Write block 3 len 20480 data: 8 9 9 10 10 18/30 Flush 19/30 Write block 544 len 24576 data: 8 9 9 10 10 10 20/30 Read block 595 len 8192 21/30 Write block 369 len 8192 data: 6 5 22/30 Read block 10 len 36864 23/30 Flush 24/30 Write block 254 len 20480 data: 13 9 8 8 7 25/30 Read block 260 len 28672 26/30 Read block 67 len 40960 27/30 Write block 42 len 28672 data: 7 7 5 5 6 6 5 28/30 Write block 334 len 4096 data: 11 29/30 Read block 386 len 36864 30/30 Read block 37 len 40960 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:12.763875143Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 137 137 137 F F F 1 020-039 77 77 77 134 134 134 F F F 2 040-059 70 70 70 125 125 125 T T T 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 75 75 75 132 132 132 F F F 6 120-139 74 74 74 131 131 131 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 77 77 77 134 134 134 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 78 78 78 136 136 136 F F F 11 220-239 78 78 78 136 136 136 F F F 12 240-259 75 75 75 132 132 132 T T T 13 260-279 75 75 75 132 132 132 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 75 75 75 132 132 132 T T T 17 340-359 72 72 72 128 128 128 F F F 18 360-379 78 78 78 138 138 138 F F F 19 380-399 78 78 78 136 136 136 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 76 76 76 133 133 133 F F F 22 440-459 72 72 72 128 128 128 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 76 76 76 133 133 133 F F F 27 540-559 78 78 78 138 138 138 F F F 28 560-579 78 78 78 136 136 136 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 78, Max flush: 138 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 137 137 137 F F F 1 020-039 77 77 77 134 134 134 F F F 2 040-059 79 79 79 139 139 139 F F F 3 060-079 71 71 71 127 127 127 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 75 75 75 132 132 132 F F F 6 120-139 74 74 74 131 131 131 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 77 77 77 134 134 134 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 78 78 78 136 136 136 F F F 11 220-239 78 78 78 136 136 136 F F F 12 240-259 79 79 79 139 139 139 F F F 13 260-279 75 75 75 132 132 132 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 79 79 79 139 139 139 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 78 78 78 138 138 138 F F F 19 380-399 78 78 78 136 136 136 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 76 76 76 133 133 133 F F F 22 440-459 72 72 72 128 128 128 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 76 76 76 133 133 133 F F F 27 540-559 78 78 78 138 138 138 F F F 28 560-579 78 78 78 136 136 136 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 79, Max flush: 139 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:16.996238226Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:17.039513101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.039928352Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.07153186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.072193107Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.07222634Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.072239871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.072254083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510} {"msg":"Crucible 6f305a32-ce5f-4497-b1d0-eb23b119b039 has session id: b61d0fa1-5e44-40b5-a69a-52448257a1be","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.072867205Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Upstairs opts: Upstairs UUID: 6f305a32-ce5f-4497-b1d0-eb23b119b039, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.072910019Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Crucible stats registered with UUID: 6f305a32-ce5f-4497-b1d0-eb23b119b039","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.072933301Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:17.072953393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073400768Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073431071Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510} {"msg":"6f305a32-ce5f-4497-b1d0-eb23b119b039 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.07351732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073540613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"0","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073577746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"0","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073605429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"1","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073634002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"1","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073663065Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"2","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073692198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"2","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073719091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073744603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073770566Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"ds_connection connected from Ok(127.0.0.1:34212)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073802819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"ds_connection connected from Ok(127.0.0.1:48789)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.073831362Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"ds_connection connected from Ok(127.0.0.1:58351)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.074071136Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.074553895Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"1","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.074590668Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"2","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.07480881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"0","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.075278218Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"1","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.075311021Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"2","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.075334583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"0","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.075875228Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.07599676Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[0]R flush_numbers[0..12]: [137, 134, 139, 127, 127, 132, 131, 128, 134, 133, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076075178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[0]R generation[0..12]: [78, 77, 79, 71, 71, 75, 74, 72, 77, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.07609474Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076109432Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[1]R flush_numbers[0..12]: [137, 134, 139, 127, 127, 132, 131, 128, 134, 133, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076128083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[1]R generation[0..12]: [78, 77, 79, 71, 71, 75, 74, 72, 77, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076152216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076179769Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[2]R flush_numbers[0..12]: [137, 134, 139, 127, 127, 132, 131, 128, 134, 133, 136, 136]","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2025-02-27T17:30:17.076210402Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[2]R generation[0..12]: [78, 77, 79, 71, 71, 75, 74, 72, 77, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076234124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076253736Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076276789Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076301081Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076323603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076337075Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"":"downstairs","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076353706Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076374238Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"0","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076398351Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"1","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076420593Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"client":"2","":"downstairs","session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076435054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076455467Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1beDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "} {"msg":"6f305a32-ce5f-4497-b1d0-eb23b119b039 is now active with session: b61d0fa1-5e44-40b5-a69a-52448257a1be","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076480989Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076501401Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510,"session_id":"b61d0fa1-5e44-40b5-a69a-52448257a1be"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:17.076521523Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 36864 data: 5 5 6 5 5 5 7 6 7 02/30 Read block 482 len 28672 03/30 Read block 539 len 32768 04/30 Read block 179 len 24576 05/30 Write block 426 len 4096 data: 7 06/30 Write block 27 len 4096 data: 7 07/30 Write block 118 len 40960 data: 13 12 14 13 11 11 11 10 8 8 08/30 Read block 361 len 24576 09/30 Write block 392 len 4096 data: 8 10/30 Write block 267 len 32768 data: 6 6 5 5 3 4 4 4 11/30 Read block 164 len 8192 12/30 Read block 444 len 20480 13/30 Read block 46 len 36864 14/30 Read block 524 len 36864 15/30 Write block 54 len 32768 data: 8 9 8 9 8 7 7 8 16/30 Read block 332 len 16384 17/30 Read block 6 len 24576 18/30 Read block 389 len 24576 19/30 Read block 168 len 28672 20/30 Write block 531 len 4096 data: 8 21/30 Read block 152 len 20480 22/30 Flush 23/30 Write block 60 len 40960 data: 8 9 6 6 5 5 4 4 5 4 24/30 Write block 367 len 20480 data: 6 5 7 6 7 25/30 Flush 26/30 Write block 451 len 28672 data: 4 7 7 6 5 6 7 27/30 Read block 496 len 32768 28/30 Read block 454 len 8192 29/30 Read block 458 len 28672 30/30 Read block 322 len 4096 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:17.098261496Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 137 137 137 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 80 80 80 140 140 140 F F F 3 060-079 80 80 80 141 141 141 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 80 80 80 140 140 140 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 80 80 80 140 140 140 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 78 78 78 136 136 136 F F F 11 220-239 78 78 78 136 136 136 F F F 12 240-259 79 79 79 139 139 139 F F F 13 260-279 80 80 80 140 140 140 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 79 79 79 139 139 139 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 80 80 80 141 141 141 F F F 19 380-399 80 80 80 140 140 140 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 80 80 80 140 140 140 F F F 22 440-459 72 72 72 128 128 128 T T T 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 80 80 80 140 140 140 F F F 27 540-559 78 78 78 138 138 138 F F F 28 560-579 78 78 78 136 136 136 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 80, Max flush: 141 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 137 137 137 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 80 80 80 140 140 140 F F F 3 060-079 80 80 80 141 141 141 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 80 80 80 140 140 140 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 80 80 80 140 140 140 F F F 9 180-199 76 76 76 133 133 133 F F F 10 200-219 78 78 78 136 136 136 F F F 11 220-239 78 78 78 136 136 136 F F F 12 240-259 79 79 79 139 139 139 F F F 13 260-279 80 80 80 140 140 140 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 79 79 79 139 139 139 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 80 80 80 141 141 141 F F F 19 380-399 80 80 80 140 140 140 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 80 80 80 140 140 140 F F F 22 440-459 81 81 81 142 142 142 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 80 80 80 140 140 140 F F F 27 540-559 78 78 78 138 138 138 F F F 28 560-579 78 78 78 136 136 136 F F F 29 580-599 73 73 73 129 129 129 F F F Max gen: 81, Max flush: 142 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.348396732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.393571178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.394362388Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.394403792Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.394435715Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.394450596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518} {"msg":"Crucible 1ed51213-d2e2-4f18-82dc-913913afc056 has session id: d17d9420-fbed-4ddc-b768-e777bb0b145f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.394924654Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Upstairs opts: Upstairs UUID: 1ed51213-d2e2-4f18-82dc-913913afc056, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.394959908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Crucible stats registered with UUID: 1ed51213-d2e2-4f18-82dc-913913afc056","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.39498397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:21.395007073Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395539597Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395564699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518} {"msg":"1ed51213-d2e2-4f18-82dc-913913afc056 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395637256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395655488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"0","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395678631Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"0","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395692732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395706763Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395719624Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"2","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395738177Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"2","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395751218Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395764509Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395777681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46670)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395946087Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57231)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.395992172Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:21.396133846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:21.396192032Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:21.396230616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.396435937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.39696405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"2","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.397003104Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"0","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.397675712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"0","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.397740408Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"2","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.398231658Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:21.39835047Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.397545258Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50258)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.397753379Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.3987576Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.399334348Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[0]R flush_numbers[0..12]: [137, 140, 140, 141, 127, 140, 140, 128, 140, 133, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.399974723Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[0]R generation[0..12]: [78, 80, 80, 80, 71, 80, 80, 72, 80, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.399993785Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400008886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[1]R flush_numbers[0..12]: [137, 140, 140, 141, 127, 140, 140, 128, 140, 133, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400022918Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[1]R generation[0..12]: [78, 80, 80, 80, 71, 80, 80, 72, 80, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400036469Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.40005016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[2]R flush_numbers[0..12]: [137, 140, 140, 141, 127, 140, 140, 128, 140, 133, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400063712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[2]R generation[0..12]: [78, 80, 80, 80, 71, 80, 80, 72, 80, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400076953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400096745Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400109866Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400127958Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400141099Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400153611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"":"downstairs","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400166332Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400178233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"0","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400191264Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"1","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400204166Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"client":"2","":"downstairs","session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400217217Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400229138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"1ed51213-d2e2-4f18-82dc-913913afc056 is now active with session: d17d9420-fbed-4ddc-b768-e777bb0b145f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.40024177Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400253471Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518,"session_id":"d17d9420-fbed-4ddc-b768-e777bb0b145f"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:31.400265502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 260 len 36864 data: 8 8 6 5 6 5 5 7 7 02/30 Read block 215 len 16384 03/30 Read block 321 len 36864 04/30 Read block 277 len 12288 05/30 Read block 316 len 28672 06/30 Read block 25 len 36864 07/30 Write block 386 len 8192 data: 9 8 08/30 Write block 120 len 12288 data: 15 14 12 09/30 Write block 592 len 8192 data: 8 9 10/30 Flush 11/30 Write block 572 len 28672 data: 9 8 7 8 6 6 6 12/30 Read block 479 len 4096 13/30 Read block 73 len 28672 14/30 Read block 102 len 32768 15/30 Read block 180 len 24576 16/30 Write block 256 len 4096 data: 9 17/30 Read block 113 len 24576 18/30 Read block 487 len 40960 19/30 Read block 249 len 28672 20/30 Write block 368 len 20480 data: 6 8 7 8 7 21/30 Write block 174 len 28672 data: 8 7 8 5 4 5 5 22/30 Flush 23/30 Write block 247 len 16384 data: 5 6 8 9 24/30 Write block 127 len 16384 data: 9 7 5 6 25/30 Read block 86 len 36864 26/30 Flush 27/30 Write block 430 len 28672 data: 6 5 3 3 4 4 6 28/30 Write block 13 len 28672 data: 9 9 11 10 9 8 9 29/30 Read block 433 len 36864 30/30 Read block 315 len 4096 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:31.420408034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 137 137 137 T T T 1 020-039 80 80 80 140 140 140 F F F 2 040-059 80 80 80 140 140 140 F F F 3 060-079 80 80 80 141 141 141 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 82 82 82 145 145 145 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 82 82 82 144 144 144 F F F 9 180-199 82 82 82 144 144 144 F F F 10 200-219 78 78 78 136 136 136 F F F 11 220-239 78 78 78 136 136 136 F F F 12 240-259 82 82 82 145 145 145 F F F 13 260-279 82 82 82 143 143 143 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 79 79 79 139 139 139 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 82 82 82 144 144 144 F F F 19 380-399 82 82 82 143 143 143 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 80 80 80 140 140 140 T T T 22 440-459 81 81 81 142 142 142 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 80 80 80 140 140 140 F F F 27 540-559 78 78 78 138 138 138 F F F 28 560-579 82 82 82 144 144 144 F F F 29 580-599 82 82 82 143 143 143 F F F Max gen: 82, Max flush: 145 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 146 146 146 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 80 80 80 140 140 140 F F F 3 060-079 80 80 80 141 141 141 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 82 82 82 145 145 145 F F F 7 140-159 72 72 72 128 128 128 F F F 8 160-179 82 82 82 144 144 144 F F F 9 180-199 82 82 82 144 144 144 F F F 10 200-219 78 78 78 136 136 136 F F F 11 220-239 78 78 78 136 136 136 F F F 12 240-259 82 82 82 145 145 145 F F F 13 260-279 82 82 82 143 143 143 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 79 79 79 139 139 139 F F F 17 340-359 72 72 72 128 128 128 F F F 18 360-379 82 82 82 144 144 144 F F F 19 380-399 82 82 82 143 143 143 F F F 20 400-419 68 68 68 118 118 118 F F F 21 420-439 83 83 83 146 146 146 F F F 22 440-459 81 81 81 142 142 142 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 F F F 25 500-519 76 76 76 133 133 133 F F F 26 520-539 80 80 80 140 140 140 F F F 27 540-559 78 78 78 138 138 138 F F F 28 560-579 82 82 82 144 144 144 F F F 29 580-599 82 82 82 143 143 143 F F F Max gen: 83, Max flush: 146 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.653918162Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:35.697971735Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.698527891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.730574503Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.731283585Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.731318919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.73133256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.731347661Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526} {"msg":"Crucible 708c848b-e4de-4f95-a783-ecf3e300797e has session id: e9be226d-c0f1-4015-9712-dd8258f847a8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.731940151Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Upstairs opts: Upstairs UUID: 708c848b-e4de-4f95-a783-ecf3e300797e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.731985906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Crucible stats registered with UUID: 708c848b-e4de-4f95-a783-ecf3e300797e","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732008978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:35.73202673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732439232Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732463524Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526} {"msg":"708c848b-e4de-4f95-a783-ecf3e300797e active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732538431Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732554873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"0","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732578166Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"0","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732591967Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"1","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732605848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"1","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.73261895Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"2","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732633031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"2","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732646162Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732659464Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732689287Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:57389)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732830881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:53952)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732864834Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:44744)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.732889197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.733574856Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"1","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.733602809Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"2","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.733990348Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"0","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.734247874Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"1","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.734472677Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"2","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.734823622Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.734848284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"0","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.73500348Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[0]R flush_numbers[0..12]: [146, 140, 140, 141, 127, 140, 145, 128, 144, 144, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735387429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[0]R generation[0..12]: [83, 80, 80, 80, 71, 80, 82, 72, 82, 82, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735408641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735423482Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[1]R flush_numbers[0..12]: [146, 140, 140, 141, 127, 140, 145, 128, 144, 144, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735440924Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[1]R generation[0..12]: [83, 80, 80, 80, 71, 80, 82, 72, 82, 82, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735454856Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735468507Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[2]R flush_numbers[0..12]: [146, 140, 140, 141, 127, 140, 145, 128, 144, 144, 136, 136]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735482048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[2]R generation[0..12]: [83, 80, 80, 80, 71, 80, 82, 72, 82, 82, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.73549535Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735509151Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735521642Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2025-02-27T17:30:35.735534314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735554416Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735567297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"":"downstairs","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735580238Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.73559185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"0","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735604951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"1","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735618022Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"client":"2","":"downstairs","session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735632754Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735644865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"708c848b-e4de-4f95-a783-ecf3e300797e is now active with session: e9be226d-c0f1-4015-9712-dd8258f847a8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735657396Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735668937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526,"session_id":"e9be226d-c0f1-4015-9712-dd8258f847a8"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:35.735680788Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 28672 data: 7 6 5 5 5 4 5 02/30 Flush 03/30 Write block 526 len 24576 data: 8 9 10 11 9 9 04/30 Read block 296 len 8192 05/30 Write block 401 len 40960 data: 5 5 4 5 5 6 6 11 11 12 06/30 Write block 512 len 40960 data: 7 8 8 7 8 7 8 11 9 9 07/30 Write block 264 len 16384 data: 7 6 6 8 08/30 Write block 362 len 4096 data: 4 09/30 Write block 439 len 40960 data: 11 9 9 8 6 6 4 5 4 4 10/30 Write block 380 len 12288 data: 6 5 6 11/30 Read block 301 len 24576 12/30 Write block 321 len 28672 data: 8 8 9 9 8 7 6 13/30 Read block 253 len 16384 14/30 Read block 269 len 32768 15/30 Write block 323 len 28672 data: 10 10 9 8 7 6 6 16/30 Flush 17/30 Write block 548 len 8192 data: 11 11 18/30 Write block 352 len 36864 data: 5 5 4 4 5 6 4 3 4 19/30 Read block 194 len 40960 20/30 Write block 212 len 40960 data: 7 8 8 9 10 11 11 11 9 9 21/30 Read block 573 len 28672 22/30 Write block 55 len 32768 data: 10 9 10 9 8 9 10 7 23/30 Flush 24/30 Write block 54 len 28672 data: 9 11 10 11 10 9 10 25/30 Write block 145 len 40960 data: 6 5 6 5 9 7 9 8 8 8 26/30 Flush 27/30 Write block 238 len 16384 data: 9 8 5 5 28/30 Write block 133 len 8192 data: 9 10 29/30 Write block 480 len 36864 data: 3 4 6 5 5 5 5 5 7 30/30 Read block 299 len 32768 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 4 3 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:35.75671536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 146 146 146 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 84 84 84 150 150 150 F F F 3 060-079 84 84 84 150 150 150 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 82 82 82 145 145 145 T T T 7 140-159 84 84 84 150 150 150 F F F 8 160-179 82 82 82 144 144 144 F F F 9 180-199 82 82 82 144 144 144 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 84 84 84 149 149 149 T T T 12 240-259 82 82 82 145 145 145 T T T 13 260-279 84 84 84 148 148 148 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 84 84 84 148 148 148 F F F 17 340-359 84 84 84 149 149 149 F F F 18 360-379 84 84 84 149 149 149 F F F 19 380-399 84 84 84 148 148 148 F F F 20 400-419 84 84 84 148 148 148 F F F 21 420-439 84 84 84 148 148 148 F F F 22 440-459 84 84 84 148 148 148 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 76 76 76 133 133 133 T T T 25 500-519 84 84 84 148 148 148 F F F 26 520-539 84 84 84 148 148 148 F F F 27 540-559 84 84 84 149 149 149 F F F 28 560-579 84 84 84 147 147 147 F F F 29 580-599 84 84 84 147 147 147 F F F Max gen: 84, Max flush: 150 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 146 146 146 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 84 84 84 150 150 150 F F F 3 060-079 84 84 84 150 150 150 F F F 4 080-099 71 71 71 127 127 127 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 85 85 85 151 151 151 F F F 7 140-159 84 84 84 150 150 150 F F F 8 160-179 82 82 82 144 144 144 F F F 9 180-199 82 82 82 144 144 144 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 85 85 85 151 151 151 F F F 12 240-259 85 85 85 151 151 151 F F F 13 260-279 84 84 84 148 148 148 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 84 84 84 148 148 148 F F F 17 340-359 84 84 84 149 149 149 F F F 18 360-379 84 84 84 149 149 149 F F F 19 380-399 84 84 84 148 148 148 F F F 20 400-419 84 84 84 148 148 148 F F F 21 420-439 84 84 84 148 148 148 F F F 22 440-459 84 84 84 148 148 148 F F F 23 460-479 72 72 72 128 128 128 F F F 24 480-499 85 85 85 151 151 151 F F F 25 500-519 84 84 84 148 148 148 F F F 26 520-539 84 84 84 148 148 148 F F F 27 540-559 84 84 84 149 149 149 F F F 28 560-579 84 84 84 147 147 147 F F F 29 580-599 84 84 84 147 147 147 F F F Max gen: 85, Max flush: 151 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.022644268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.06777651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.068540377Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.068592063Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.068611034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.068629316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534} {"msg":"Crucible 74648438-d8ed-4257-865c-f44120879331 has session id: 4b78f8f0-2757-48e6-a102-47adeab043a9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.069241778Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Upstairs opts: Upstairs UUID: 74648438-d8ed-4257-865c-f44120879331, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.069281182Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Crucible stats registered with UUID: 74648438-d8ed-4257-865c-f44120879331","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.069304154Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:40.069323976Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.069836128Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.069866491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534} {"msg":"74648438-d8ed-4257-865c-f44120879331 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.069928177Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.069960701Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"0","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.069998284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"0","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070025167Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"1","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070061301Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"1","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070092174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"2","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070120407Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"2","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070147539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070175922Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070199935Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:51110)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070228297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56699)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.07025597Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42688)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070286403Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.070989824Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"2","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.071027618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"0","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.071672283Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"2","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.071709147Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"0","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.072317828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.072352742Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.072588286Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"1","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073155033Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"1","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"[0]R flush_numbers[0..12]: [146, 140, 150, 150, 127, 140, 151, 150, 144, 144, 149, 151]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073679916Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"[0]R generation[0..12]: [83, 80, 84, 84, 71, 80, 85, 84, 82, 82, 84, 85]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073702178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.07372057Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"[1]R flush_numbers[0..12]: [146, 140, 150, 150, 127, 140, 151, 150, 144, 144, 149, 151]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073737811Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"[1]R generation[0..12]: [83, 80, 84, 84, 71, 80, 85, 84, 82, 82, 84, 85]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073754423Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073771515Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"[2]R flush_numbers[0..12]: [146, 140, 150, 150, 127, 140, 151, 150, 144, 144, 149, 151]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073787987Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"[2]R generation[0..12]: [83, 80, 84, 84, 71, 80, 85, 84, 82, 82, 84, 85]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073804318Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"}Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073833341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073862244Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073892517Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.07392677Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.073959194Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"":"downstairs","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.074014889Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.074040902Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"0","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.074058434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"1","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.074074816Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"client":"2","":"downstairs","session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.074091157Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.074106509Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"74648438-d8ed-4257-865c-f44120879331 is now active with session: 4b78f8f0-2757-48e6-a102-47adeab043a9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.07412216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.074136812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534,"session_id":"4b78f8f0-2757-48e6-a102-47adeab043a9"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:40.074151613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 332 len 12288 data: 10 9 12 02/30 Read block 177 len 20480 03/30 Read block 66 len 20480 04/30 Write block 466 len 24576 data: 11 11 10 9 9 11 05/30 Write block 221 len 12288 data: 10 11 11 06/30 Write block 583 len 24576 data: 5 6 5 5 5 6 07/30 Flush 08/30 Write block 33 len 20480 data: 6 6 6 6 7 09/30 Write block 360 len 40960 data: 5 5 5 5 5 5 5 7 7 9 10/30 Write block 441 len 8192 data: 10 9 11/30 Write block 364 len 12288 data: 6 6 6 12/30 Read block 263 len 36864 13/30 Write block 524 len 24576 data: 8 8 9 10 11 12 14/30 Write block 474 len 20480 data: 4 4 4 2 2 15/30 Read block 305 len 16384 16/30 Write block 548 len 40960 data: 12 12 10 9 8 8 5 5 6 7 17/30 Read block 412 len 8192 18/30 Flush 19/30 Write block 74 len 28672 data: 6 8 8 6 7 8 8 20/30 Write block 35 len 36864 data: 7 7 8 7 8 7 7 8 8 21/30 Read block 539 len 4096 22/30 Read block 185 len 16384 23/30 Read block 584 len 40960 24/30 Flush 25/30 Write block 82 len 36864 data: 5 7 7 8 11 10 9 10 9 26/30 Read block 393 len 32768 27/30 Write block 166 len 40960 data: 5 5 6 6 7 6 6 6 9 8 28/30 Write block 514 len 20480 data: 9 8 9 8 9 29/30 Read block 36 len 40960 30/30 Write block 262 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:40.094221307Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 146 146 146 F F F 1 020-039 86 86 86 154 154 154 F F F 2 040-059 86 86 86 154 154 154 F F F 3 060-079 86 86 86 154 154 154 F F F 4 080-099 86 86 86 154 154 154 T T T 5 100-119 80 80 80 140 140 140 F F F 6 120-139 85 85 85 151 151 151 F F F 7 140-159 84 84 84 150 150 150 F F F 8 160-179 82 82 82 144 144 144 T T T 9 180-199 82 82 82 144 144 144 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 85 85 85 151 151 151 F F F 13 260-279 84 84 84 148 148 148 T T T 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 86 86 86 152 152 152 F F F 17 340-359 84 84 84 149 149 149 F F F 18 360-379 86 86 86 153 153 153 F F F 19 380-399 84 84 84 148 148 148 F F F 20 400-419 84 84 84 148 148 148 F F F 21 420-439 84 84 84 148 148 148 F F F 22 440-459 86 86 86 153 153 153 F F F 23 460-479 86 86 86 153 153 153 F F F 24 480-499 85 85 85 151 151 151 F F F 25 500-519 84 84 84 148 148 148 T T T 26 520-539 86 86 86 153 153 153 F F F 27 540-559 86 86 86 153 153 153 F F F 28 560-579 84 84 84 147 147 147 F F F 29 580-599 86 86 86 152 152 152 F F F Max gen: 86, Max flush: 154 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 146 146 146 F F F 1 020-039 86 86 86 154 154 154 F F F 2 040-059 86 86 86 154 154 154 F F F 3 060-079 86 86 86 154 154 154 F F F 4 080-099 87 87 87 155 155 155 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 85 85 85 151 151 151 F F F 7 140-159 84 84 84 150 150 150 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 82 82 82 144 144 144 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 85 85 85 151 151 151 F F F 13 260-279 87 87 87 155 155 155 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 86 86 86 152 152 152 F F F 17 340-359 84 84 84 149 149 149 F F F 18 360-379 86 86 86 153 153 153 F F F 19 380-399 84 84 84 148 148 148 F F F 20 400-419 84 84 84 148 148 148 F F F 21 420-439 84 84 84 148 148 148 F F F 22 440-459 86 86 86 153 153 153 F F F 23 460-479 86 86 86 153 153 153 F F F 24 480-499 85 85 85 151 151 151 F F F 25 500-519 87 87 87 155 155 155 F F F 26 520-539 86 86 86 153 153 153 F F F 27 540-559 86 86 86 153 153 153 F F F 28 560-579 84 84 84 147 147 147 F F F 29 580-599 86 86 86 152 152 152 F F F Max gen: 87, Max flush: 155 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.328793754Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.375499414Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.376323107Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.376370072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.376384854Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.376400155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542} {"msg":"Crucible 435bce0a-1c01-4fdd-b50f-544691d5414e has session id: ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377160152Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Upstairs opts: Upstairs UUID: 435bce0a-1c01-4fdd-b50f-544691d5414e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377188404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Crucible stats registered with UUID: 435bce0a-1c01-4fdd-b50f-544691d5414e","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377203636Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:44.377216467Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377654591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377678344Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542} {"msg":"435bce0a-1c01-4fdd-b50f-544691d5414e active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377758442Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377775934Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"0","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377802506Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"0","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377817758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"1","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.37783234Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"1","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377846161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"2","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377860112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"2","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377873354Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377886995Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.377900896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"ds_connection connected from Ok(127.0.0.1:52094)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.378183345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"ds_connection connected from Ok(127.0.0.1:45559)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.378304697Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"ds_connection connected from Ok(127.0.0.1:63018)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.378342591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.378937421Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"2","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.378978065Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"0","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.379642502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"0","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.379670235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"2","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.379981706Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"1","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.380233412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.380343633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.380609749Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"1","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[0]R flush_numbers[0..12]: [146, 154, 154, 154, 155, 140, 151, 150, 155, 144, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381149334Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[0]R generation[0..12]: [83, 86, 86, 86, 87, 80, 85, 84, 87, 82, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381169276Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381183898Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[1]R flush_numbers[0..12]: [146, 154, 154, 154, 155, 140, 151, 150, 155, 144, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381197649Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[1]R generation[0..12]: [83, 86, 86, 86, 87, 80, 85, 84, 87, 82, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.38121075Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381224652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[2]R flush_numbers[0..12]: [146, 154, 154, 154, 155, 140, 151, 150, 155, 144, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381238163Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[2]R generation[0..12]: [83, 86, 86, 86, 87, 80, 85, 84, 87, 82, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381254565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:30:44.381268726Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381286248Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381299199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.38131173Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381324392Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"":"downstairs","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381337673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381349664Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"0","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381363156Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"1","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381376047Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"client":"2","":"downstairs","session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381388988Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.38141299Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"435bce0a-1c01-4fdd-b50f-544691d5414e is now active with session: ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381427052Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381439093Z","hostnameDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542,"session_id":"ee87afbd-27d7-46cf-bb9f-bcbc1bc1f743"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:44.381454935Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 286 len 12288 data: 5 6 5 02/30 Write block 363 len 20480 data: 6 7 7 7 8 03/30 Write block 134 len 40960 data: 11 10 11 10 10 12 14 14 12 11 04/30 Write block 267 len 24576 data: 9 8 6 6 4 5 05/30 Write block 12 len 20480 data: 7 10 10 12 11 06/30 Read block 283 len 4096 07/30 Read block 77 len 8192 08/30 Write block 69 len 32768 data: 5 5 6 5 6 7 9 9 09/30 Read block 427 len 8192 10/30 Flush 11/30 Write block 395 len 20480 data: 10 10 9 5 4 12/30 Write block 349 len 40960 data: 5 6 5 6 6 5 5 6 7 5 13/30 Write block 33 len 16384 data: 7 7 8 8 14/30 Read block 142 len 20480 15/30 Read block 32 len 24576 16/30 Write block 354 len 8192 data: 6 6 17/30 Read block 475 len 32768 18/30 Flush 19/30 Write block 418 len 24576 data: 8 7 6 6 7 5 20/30 Read block 198 len 24576 21/30 Read block 78 len 28672 22/30 Flush 23/30 Write block 191 len 36864 data: 4 5 8 8 10 7 9 7 9 24/30 Read block 263 len 24576 25/30 Write block 452 len 12288 data: 8 8 7 26/30 Read block 548 len 32768 27/30 Read block 344 len 4096 28/30 Write block 398 len 12288 data: 6 5 5 29/30 Read block 33 len 8192 30/30 Read block 116 len 4096 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:44.400673603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 156 156 156 F F F 1 020-039 88 88 88 157 157 157 F F F 2 040-059 86 86 86 154 154 154 F F F 3 060-079 88 88 88 156 156 156 F F F 4 080-099 87 87 87 155 155 155 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 88 88 88 156 156 156 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 82 82 82 144 144 144 T T T 10 200-219 84 84 84 149 149 149 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 85 85 85 151 151 151 F F F 13 260-279 88 88 88 156 156 156 F F F 14 280-299 88 88 88 156 156 156 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 86 86 86 152 152 152 F F F 17 340-359 88 88 88 157 157 157 F F F 18 360-379 88 88 88 156 156 156 F F F 19 380-399 88 88 88 157 157 157 T T T 20 400-419 88 88 88 158 158 158 T T T 21 420-439 88 88 88 158 158 158 F F F 22 440-459 86 86 86 153 153 153 T T T 23 460-479 86 86 86 153 153 153 F F F 24 480-499 85 85 85 151 151 151 F F F 25 500-519 87 87 87 155 155 155 F F F 26 520-539 86 86 86 153 153 153 F F F 27 540-559 86 86 86 153 153 153 F F F 28 560-579 84 84 84 147 147 147 F F F 29 580-599 86 86 86 152 152 152 F F F Max gen: 88, Max flush: 158 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 156 156 156 F F F 1 020-039 88 88 88 157 157 157 F F F 2 040-059 86 86 86 154 154 154 F F F 3 060-079 88 88 88 156 156 156 F F F 4 080-099 87 87 87 155 155 155 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 88 88 88 156 156 156 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 85 85 85 151 151 151 F F F 13 260-279 88 88 88 156 156 156 F F F 14 280-299 88 88 88 156 156 156 F F F 15 300-319 78 78 78 136 136 136 F F F 16 320-339 86 86 86 152 152 152 F F F 17 340-359 88 88 88 157 157 157 F F F 18 360-379 88 88 88 156 156 156 F F F 19 380-399 89 89 89 159 159 159 F F F 20 400-419 89 89 89 159 159 159 F F F 21 420-439 88 88 88 158 158 158 F F F 22 440-459 89 89 89 159 159 159 F F F 23 460-479 86 86 86 153 153 153 F F F 24 480-499 85 85 85 151 151 151 F F F 25 500-519 87 87 87 155 155 155 F F F 26 520-539 86 86 86 153 153 153 F F F 27 540-559 86 86 86 153 153 153 F F F 28 560-579 84 84 84 147 147 147 F F F 29 580-599 86 86 86 152 152 152 F F F Max gen: 89, Max flush: 159 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.641371217Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.688303051Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.689118253Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.689162107Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.689177309Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.68919215Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550} {"msg":"Crucible 9b33335f-b306-4658-b75c-673a3a1a4447 has session id: 8b3310d6-13dd-4104-ad6c-451d921f9d97","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.689798162Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Upstairs opts: Upstairs UUID: 9b33335f-b306-4658-b75c-673a3a1a4447, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.689836715Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Crucible stats registered with UUID: 9b33335f-b306-4658-b75c-673a3a1a4447","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.689859658Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:58.68987919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690365529Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690389161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550} {"msg":"9b33335f-b306-4658-b75c-673a3a1a4447 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690481531Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690514754Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"0","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690550797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"0","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690591402Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"1","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690621314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"1","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690646027Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"2","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690668759Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"2","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690692412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690718294Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690749458Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"ds_connection connected from Ok(127.0.0.1:60331)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.69077371Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"ds_connection connected from Ok(127.0.0.1:49922)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690797352Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"ds_connection connected from Ok(127.0.0.1:44753)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.690946137Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.691503074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"0","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.691531056Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"1","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.692176201Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"1","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.692210515Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"0","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.69285413Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.692878782Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.693493574Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"2","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.693995475Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"2","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[0]R flush_numbers[0..12]: [156, 157, 154, 156, 155, 140, 156, 156, 155, 159, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694518407Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[0]R generation[0..12]: [88, 88, 86, 88, 87, 80, 88, 88, 87, 89, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694537129Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694554631Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[1]R flush_numbers[0..12]: [156, 157, 154, 156, 155, 140, 156, 156, 155, 159, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694582334Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[1]R generation[0..12]: [88, 88, 86, 88, 87, 80, 88, 88, 87, 89, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694596895Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694610357Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[2]R flush_numbers[0..12]: [156, 157, 154, 156, 155, 140, 156, 156, 155, 159, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694623908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[2]R generation[0..12]: [88, 88, 86, 88, 87, 80, 88, 88, 87, 89, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.69463687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694650591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:30:58.694663192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694678784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694691375Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694704206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"":"downstairs","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694716907Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694728439Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"0","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.69474139Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"1","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694756091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"client":"2","":"downstairs","session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:30:58.694769003Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694783304Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"9b33335f-b306-4658-b75c-673a3a1a4447 is now active with session: 8b3310d6-13dd-4104-ad6c-451d921f9d97","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694795626Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694807257Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550,"session_id":"8b3310d6-13dd-4104-ad6c-451d921f9d97"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:30:58.694832609Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 79 len 32768 data: 9 9 7 6 8 8 9 12 02/30 Read block 280 len 40960 03/30 Read block 541 len 12288 04/30 Read block 373 len 32768 05/30 Write block 331 len 8192 data: 7 11 06/30 Read block 578 len 8192 07/30 Read block 535 len 12288 08/30 Write block 244 len 40960 data: 6 6 6 6 7 9 10 11 13 16 09/30 Read block 469 len 4096 10/30 Read block 104 len 20480 11/30 Read block 223 len 8192 12/30 Read block 579 len 12288 13/30 Flush 14/30 Write block 314 len 36864 data: 5 6 6 8 8 9 8 9 9 15/30 Write block 121 len 12288 data: 15 13 12 16/30 Write block 383 len 24576 data: 8 8 10 10 9 9 17/30 Write block 37 len 32768 data: 9 8 9 8 8 9 9 6 18/30 Write block 8 len 4096 data: 11 19/30 Read block 259 len 36864 20/30 Write block 317 len 32768 data: 9 9 10 9 10 10 11 11 21/30 Read block 2 len 8192 22/30 Write block 562 len 16384 data: 4 4 5 7 23/30 Write block 245 len 24576 data: 7 7 7 8 10 11 24/30 Read block 158 len 20480 25/30 Write block 473 len 40960 data: 6 5 5 5 3 3 2 4 5 7 26/30 Read block 470 len 8192 27/30 Read block 343 len 8192 28/30 Read block 246 len 36864 29/30 Write block 51 len 8192 data: 5 6 30/30 Read block 291 len 40960 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 17 17 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:30:58.715227056Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 156 156 156 T T T 1 020-039 88 88 88 157 157 157 T T T 2 040-059 86 86 86 154 154 154 T T T 3 060-079 90 90 90 160 160 160 F F F 4 080-099 90 90 90 160 160 160 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 88 88 88 156 156 156 T T T 7 140-159 88 88 88 156 156 156 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 90 90 90 160 160 160 T T T 13 260-279 88 88 88 156 156 156 F F F 14 280-299 88 88 88 156 156 156 F F F 15 300-319 78 78 78 136 136 136 T T T 16 320-339 90 90 90 160 160 160 T T T 17 340-359 88 88 88 157 157 157 F F F 18 360-379 88 88 88 156 156 156 F F F 19 380-399 89 89 89 159 159 159 T T T 20 400-419 89 89 89 159 159 159 F F F 21 420-439 88 88 88 158 158 158 F F F 22 440-459 89 89 89 159 159 159 F F F 23 460-479 86 86 86 153 153 153 T T T 24 480-499 85 85 85 151 151 151 T T T 25 500-519 87 87 87 155 155 155 F F F 26 520-539 86 86 86 153 153 153 F F F 27 540-559 86 86 86 153 153 153 F F F 28 560-579 84 84 84 147 147 147 T T T 29 580-599 86 86 86 152 152 152 F F F Max gen: 90, Max flush: 160 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 161 161 161 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 91 91 91 161 161 161 F F F 3 060-079 90 90 90 160 160 160 F F F 4 080-099 90 90 90 160 160 160 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 86 86 86 152 152 152 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 88 88 88 156 156 156 F F F 14 280-299 88 88 88 156 156 156 F F F 15 300-319 91 91 91 161 161 161 F F F 16 320-339 91 91 91 161 161 161 F F F 17 340-359 88 88 88 157 157 157 F F F 18 360-379 88 88 88 156 156 156 F F F 19 380-399 91 91 91 161 161 161 F F F 20 400-419 89 89 89 159 159 159 F F F 21 420-439 88 88 88 158 158 158 F F F 22 440-459 89 89 89 159 159 159 F F F 23 460-479 91 91 91 161 161 161 F F F 24 480-499 91 91 91 161 161 161 F F F 25 500-519 87 87 87 155 155 155 F F F 26 520-539 86 86 86 153 153 153 F F F 27 540-559 86 86 86 153 153 153 F F F 28 560-579 91 91 91 161 161 161 F F F 29 580-599 86 86 86 152 152 152 F F F Max gen: 91, Max flush: 161 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:12.993500397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.038168302Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.039080944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.039134259Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.039155741Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.039178384Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558} {"msg":"Crucible d7f4ad1b-a121-4a79-b170-346cfb6d1e0d has session id: 888bc780-1519-44fe-81a5-02b1cf6e8ace","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.039690846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Upstairs opts: Upstairs UUID: d7f4ad1b-a121-4a79-b170-346cfb6d1e0d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.039729069Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Crucible stats registered with UUID: d7f4ad1b-a121-4a79-b170-346cfb6d1e0d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.039768713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:13.039792376Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040217619Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040249602Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558} {"msg":"d7f4ad1b-a121-4a79-b170-346cfb6d1e0d active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040308598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.04033369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"0","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040362753Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"0","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040377475Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040391526Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040404478Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"2","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040419229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"2","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040468604Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040492366Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040506968Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"ds_connection connected from Ok(127.0.0.1:37013)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040564434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"ds_connection connected from Ok(127.0.0.1:64020)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.040599767Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:13.04063159Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:13.040662073Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:13.040690876Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.04082522Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.041298278Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"2","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.041326311Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"0","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.041950734Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"0","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.041978186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"2","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.042643883Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:13.042669016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.042666276Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"ds_connection connected from Ok(127.0.0.1:56586)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.042851095Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.043822453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.044353816Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[0]R flush_numbers[0..12]: [161, 161, 161, 160, 160, 140, 161, 156, 155, 159, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.044973538Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[0]R generation[0..12]: [91, 91, 91, 90, 90, 80, 91, 88, 87, 89, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045018263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045036095Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[1]R flush_numbers[0..12]: [161, 161, 161, 160, 160, 140, 161, 156, 155, 159, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045050646Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[1]R generation[0..12]: [91, 91, 91, 90, 90, 80, 91, 88, 87, 89, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045071748Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.04508638Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[2]R flush_numbers[0..12]: [161, 161, 161, 160, 160, 140, 161, 156, 155, 159, 149, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045100211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[2]R generation[0..12]: [91, 91, 91, 90, 90, 80, 91, 88, 87, 89, 84, 86]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045113713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045127714Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045144646Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045157657Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045170328Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.04518297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"":"downstairs","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045195931Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045207652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"0","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045221214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"1","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045234195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"client":"2","":"downstairs","session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045247266Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045259297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"d7f4ad1b-a121-4a79-b170-346cfb6d1e0d is now active with session: 888bc780-1519-44fe-81a5-02b1cf6e8ace","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045273899Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.04528561Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558,"session_id":"888bc780-1519-44fe-81a5-02b1cf6e8ace"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:23.045297551Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 404 len 8192 data: 6 6 02/30 Read block 552 len 40960 03/30 Write block 357 len 16384 data: 8 6 4 6 04/30 Write block 63 len 36864 data: 7 6 6 5 5 6 6 6 7 05/30 Read block 102 len 4096 06/30 Write block 286 len 28672 data: 6 7 6 5 5 6 5 07/30 Write block 230 len 8192 data: 7 7 08/30 Read block 368 len 28672 09/30 Read block 532 len 20480 10/30 Write block 317 len 24576 data: 10 10 11 10 11 11 11/30 Read block 575 len 32768 12/30 Write block 315 len 40960 data: 7 7 11 11 12 11 12 12 12 12 13/30 Read block 564 len 36864 14/30 Write block 496 len 28672 data: 5 5 6 7 6 6 5 15/30 Read block 246 len 12288 16/30 Read block 230 len 24576 17/30 Read block 230 len 8192 18/30 Read block 265 len 32768 19/30 Read block 258 len 36864 20/30 Read block 353 len 16384 21/30 Read block 578 len 4096 22/30 Read block 495 len 36864 23/30 Write block 377 len 8192 data: 6 6 24/30 Read block 590 len 20480 25/30 Flush 26/30 Write block 526 len 8192 data: 10 11 27/30 Read block 70 len 36864 28/30 Write block 381 len 8192 data: 6 7 29/30 Read block 123 len 8192 30/30 Read block 183 len 36864 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:23.067250055Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 161 161 161 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 91 91 91 161 161 161 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 90 90 90 160 160 160 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 92 92 92 162 162 162 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 88 88 88 156 156 156 F F F 14 280-299 92 92 92 162 162 162 F F F 15 300-319 92 92 92 162 162 162 F F F 16 320-339 92 92 92 162 162 162 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 92 92 92 162 162 162 F F F 19 380-399 91 91 91 161 161 161 T T T 20 400-419 92 92 92 162 162 162 F F F 21 420-439 88 88 88 158 158 158 F F F 22 440-459 89 89 89 159 159 159 F F F 23 460-479 91 91 91 161 161 161 F F F 24 480-499 92 92 92 162 162 162 F F F 25 500-519 92 92 92 162 162 162 F F F 26 520-539 86 86 86 153 153 153 T T T 27 540-559 86 86 86 153 153 153 F F F 28 560-579 91 91 91 161 161 161 F F F 29 580-599 86 86 86 152 152 152 F F F Max gen: 92, Max flush: 162 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 161 161 161 F F F 1 020-039 91 91 91 161 161 161 F F F 2 040-059 91 91 91 161 161 161 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 90 90 90 160 160 160 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 87 87 87 155 155 155 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 92 92 92 162 162 162 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 88 88 88 156 156 156 F F F 14 280-299 92 92 92 162 162 162 F F F 15 300-319 92 92 92 162 162 162 F F F 16 320-339 92 92 92 162 162 162 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 92 92 92 162 162 162 F F F 19 380-399 93 93 93 163 163 163 F F F 20 400-419 92 92 92 162 162 162 F F F 21 420-439 88 88 88 158 158 158 F F F 22 440-459 89 89 89 159 159 159 F F F 23 460-479 91 91 91 161 161 161 F F F 24 480-499 92 92 92 162 162 162 F F F 25 500-519 92 92 92 162 162 162 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 86 86 86 153 153 153 F F F 28 560-579 91 91 91 161 161 161 F F F 29 580-599 86 86 86 152 152 152 F F F Max gen: 93, Max flush: 163 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.298230309Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.342724177Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.343506006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.34355444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.343579533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.343597805Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566} {"msg":"Crucible da9e8a94-24e6-49ed-8f3f-def69b55189f has session id: 6efff700-7a57-4ead-a36d-011bf7e0d782","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344174123Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Upstairs opts: Upstairs UUID: da9e8a94-24e6-49ed-8f3f-def69b55189f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344219718Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Crucible stats registered with UUID: da9e8a94-24e6-49ed-8f3f-def69b55189f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.34424583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:27.344262292Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344711347Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.34473399Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566} {"msg":"da9e8a94-24e6-49ed-8f3f-def69b55189f active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344811847Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344828889Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"0","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344850981Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"0","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344865543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"1","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344889885Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"1","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344914048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"2","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344951001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"2","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.344982655Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.345009507Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.34503583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"ds_connection connected from Ok(127.0.0.1:60521)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.345061752Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"ds_connection connected from Ok(127.0.0.1:57784)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.345087465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"ds_connection connected from Ok(127.0.0.1:46154)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.345113018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.345746432Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"1","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.345773804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"0","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.346414109Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"0","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.346447112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"1","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.347061374Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.347086107Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.347587957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"2","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348103229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"2","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[0]R flush_numbers[0..12]: [161, 161, 161, 162, 160, 140, 161, 156, 155, 159, 149, 162]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348618401Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[0]R generation[0..12]: [91, 91, 91, 92, 90, 80, 91, 88, 87, 89, 84, 92]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348642334Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348657345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[1]R flush_numbers[0..12]: [161, 161, 161, 162, 160, 140, 161, 156, 155, 159, 149, 162]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348671437Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[1]R generation[0..12]: [91, 91, 91, 92, 90, 80, 91, 88, 87, 89, 84, 92]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348686088Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.34869985Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[2]R flush_numbers[0..12]: [161, 161, 161, 162, 160, 140, 161, 156, 155, 159, 149, 162]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348713281Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[2]R generation[0..12]: [91, 91, 91, 92, 90, 80, 91, 88, 87, 89, 84, 92]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348726272Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348739623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348760576Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348785038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348809161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348831923Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"":"downstairs","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348855275Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Transition from WaitQuorum to Active","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348885748Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"0","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348912601Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"1","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348949015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"client":"2","":"downstairs","session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348964266Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348977098Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"da9e8a94-24e6-49ed-8f3f-def69b55189f is now active with session: 6efff700-7a57-4ead-a36d-011bf7e0d782","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.348989599Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.349009271Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566,"session_id":"6efff700-7a57-4ead-a36d-011bf7e0d782"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:27.349026623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 220 len 36864 data: 10 11 12 12 12 13 10 8 8 02/30 Read block 269 len 20480 03/30 Write block 46 len 28672 data: 7 7 6 5 5 6 7 04/30 Read block 261 len 20480 05/30 Write block 265 len 20480 data: 7 7 10 9 7 06/30 Read block 20 len 36864 07/30 Flush 08/30 Write block 510 len 28672 data: 6 5 8 9 10 9 10 09/30 Write block 168 len 8192 data: 7 7 10/30 Write block 584 len 24576 data: 7 6 6 6 7 7 11/30 Write block 503 len 32768 data: 5 4 5 5 4 4 7 7 12/30 Read block 446 len 28672 13/30 Read block 566 len 28672 14/30 Read block 144 len 28672 15/30 Read block 325 len 16384 16/30 Read block 362 len 16384 17/30 Read block 3 len 24576 18/30 Write block 318 len 16384 data: 12 13 12 13 19/30 Flush 20/30 Write block 417 len 28672 data: 8 9 8 7 7 8 6 21/30 Read block 190 len 40960 22/30 Write block 25 len 28672 data: 6 6 8 7 7 6 6 23/30 Read block 417 len 28672 24/30 Read block 364 len 8192 25/30 Read block 442 len 12288 26/30 Read block 445 len 20480 27/30 Write block 554 len 24576 data: 6 6 7 8 7 8 28/30 Read block 81 len 16384 29/30 Read block 389 len 36864 30/30 Read block 44 len 4096 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:27.370366655Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 161 161 161 F F F 1 020-039 91 91 91 161 161 161 T T T 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 90 90 90 160 160 160 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 94 94 94 164 164 164 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 92 92 92 162 162 162 F F F 15 300-319 94 94 94 165 165 165 F F F 16 320-339 94 94 94 165 165 165 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 92 92 92 162 162 162 F F F 19 380-399 93 93 93 163 163 163 F F F 20 400-419 92 92 92 162 162 162 T T T 21 420-439 88 88 88 158 158 158 T T T 22 440-459 89 89 89 159 159 159 F F F 23 460-479 91 91 91 161 161 161 F F F 24 480-499 92 92 92 162 162 162 F F F 25 500-519 94 94 94 165 165 165 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 86 86 86 153 153 153 T T T 28 560-579 91 91 91 161 161 161 F F F 29 580-599 94 94 94 165 165 165 F F F Max gen: 94, Max flush: 165 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 161 161 161 F F F 1 020-039 95 95 95 166 166 166 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 90 90 90 160 160 160 F F F 5 100-119 80 80 80 140 140 140 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 84 84 84 149 149 149 F F F 11 220-239 94 94 94 164 164 164 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 92 92 92 162 162 162 F F F 15 300-319 94 94 94 165 165 165 F F F 16 320-339 94 94 94 165 165 165 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 92 92 92 162 162 162 F F F 19 380-399 93 93 93 163 163 163 F F F 20 400-419 95 95 95 166 166 166 F F F 21 420-439 95 95 95 166 166 166 F F F 22 440-459 89 89 89 159 159 159 F F F 23 460-479 91 91 91 161 161 161 F F F 24 480-499 92 92 92 162 162 162 F F F 25 500-519 94 94 94 165 165 165 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 95 95 95 166 166 166 F F F 28 560-579 91 91 91 161 161 161 F F F 29 580-599 94 94 94 165 165 165 F F F Max gen: 95, Max flush: 166 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.601611735Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.646358828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.647088982Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.647128986Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.647143478Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.647157999Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574} {"msg":"Crucible 43655a83-b955-4ae0-a9ed-e438f8bb42d6 has session id: cebea823-0d2a-49f5-a37e-06937c657e8a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.64786457Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Upstairs opts: Upstairs UUID: 43655a83-b955-4ae0-a9ed-e438f8bb42d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.647910865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Crucible stats registered with UUID: 43655a83-b955-4ae0-a9ed-e438f8bb42d6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.647939528Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:31.64796325Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648532858Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648567661Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574} {"msg":"43655a83-b955-4ae0-a9ed-e438f8bb42d6 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648621777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.64865334Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"0","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648689463Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"0","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648721937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.64874991Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648776232Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"2","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648804505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"2","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648826868Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.64884941Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648871812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39977)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648915516Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:31.64895051Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38297)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.648981243Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:31.649008596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:31.649036618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.649162241Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.649659231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"2","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.649697235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"0","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.650321198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"0","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.650347911Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"2","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.650954982Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:31.650977404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.650107796Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42941)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.650232719Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.651520589Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.65212395Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[0]R flush_numbers[0..12]: [161, 166, 164, 162, 160, 140, 161, 156, 165, 159, 149, 164]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652738522Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[0]R generation[0..12]: [91, 95, 94, 92, 90, 80, 91, 88, 94, 89, 84, 94]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652767525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652786427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[1]R flush_numbers[0..12]: [161, 166, 164, 162, 160, 140, 161, 156, 165, 159, 149, 164]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652809219Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[1]R generation[0..12]: [91, 95, 94, 92, 90, 80, 91, 88, 94, 89, 84, 94]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652827201Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652845913Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[2]R flush_numbers[0..12]: [161, 166, 164, 162, 160, 140, 161, 156, 165, 159, 149, 164]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652863565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[2]R generation[0..12]: [91, 95, 94, 92, 90, 80, 91, 88, 94, 89, 84, 94]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-02-27T17:31:41.652880686Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652912979Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652943583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.652973876Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653002728Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.6530201Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"":"downstairs","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653036412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653051013Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"0","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653067015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"client":"1","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653082807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 pid":1574,"client":"2","":"downstairs","session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653102729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653121251Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"43655a83-b955-4ae0-a9ed-e438f8bb42d6 is now active with session: cebea823-0d2a-49f5-a37e-06937c657e8a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653137062Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653151714Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574,"session_id":"cebea823-0d2a-49f5-a37e-06937c657e8a"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:41.653166465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 101 len 28672 data: 4 4 5 5 7 7 7 02/30 Read block 171 len 4096 03/30 Read block 502 len 16384 04/30 Read block 352 len 4096 05/30 Read block 228 len 36864 06/30 Write block 581 len 24576 data: 6 6 6 8 7 7 07/30 Read block 36 len 20480 08/30 Write block 392 len 12288 data: 9 9 9 09/30 Write block 514 len 8192 data: 11 10 10/30 Read block 149 len 20480 11/30 Write block 229 len 12288 data: 6 8 8 12/30 Write block 424 len 32768 data: 4 7 8 7 7 6 7 6 13/30 Write block 9 len 36864 data: 9 10 7 8 11 11 13 12 10 14/30 Write block 31 len 28672 data: 7 7 8 8 9 9 10 15/30 Read block 389 len 4096 16/30 Read block 465 len 16384 17/30 Flush 18/30 Write block 239 len 4096 data: 9 19/30 Write block 215 len 4096 data: 10 20/30 Read block 194 len 36864 21/30 Read block 46 len 36864 22/30 Write block 550 len 4096 data: 11 23/30 Flush 24/30 Write block 287 len 24576 data: 8 7 6 6 7 6 25/30 Read block 231 len 32768 26/30 Write block 424 len 36864 data: 5 8 9 8 8 7 8 7 4 27/30 Write block 8 len 8192 data: 12 10 28/30 Write block 111 len 12288 data: 8 9 7 29/30 Write block 371 len 40960 data: 9 8 6 6 5 5 7 7 7 7 30/30 Write block 560 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:41.67126349Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 167 167 167 T T T 1 020-039 96 96 96 167 167 167 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 90 90 90 160 160 160 F F F 5 100-119 96 96 96 167 167 167 T T T 6 120-139 91 91 91 161 161 161 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 96 96 96 168 168 168 F F F 11 220-239 96 96 96 168 168 168 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 92 92 92 162 162 162 T T T 15 300-319 94 94 94 165 165 165 F F F 16 320-339 94 94 94 165 165 165 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 92 92 92 162 162 162 T T T 19 380-399 96 96 96 167 167 167 T T T 20 400-419 95 95 95 166 166 166 F F F 21 420-439 96 96 96 167 167 167 T T T 22 440-459 89 89 89 159 159 159 F F F 23 460-479 91 91 91 161 161 161 F F F 24 480-499 92 92 92 162 162 162 F F F 25 500-519 96 96 96 167 167 167 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 96 96 96 168 168 168 F F F 28 560-579 91 91 91 161 161 161 T T T 29 580-599 96 96 96 167 167 167 F F F Max gen: 96, Max flush: 168 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 169 169 169 F F F 1 020-039 96 96 96 167 167 167 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 90 90 90 160 160 160 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 88 88 88 156 156 156 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 96 96 96 168 168 168 F F F 11 220-239 96 96 96 168 168 168 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 97 97 97 169 169 169 F F F 15 300-319 94 94 94 165 165 165 F F F 16 320-339 94 94 94 165 165 165 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 97 97 97 169 169 169 F F F 19 380-399 97 97 97 169 169 169 F F F 20 400-419 95 95 95 166 166 166 F F F 21 420-439 97 97 97 169 169 169 F F F 22 440-459 89 89 89 159 159 159 F F F 23 460-479 91 91 91 161 161 161 F F F 24 480-499 92 92 92 162 162 162 F F F 25 500-519 96 96 96 167 167 167 F F F 26 520-539 93 93 93 163 163 163 F F F 27 540-559 96 96 96 168 168 168 F F F 28 560-579 97 97 97 169 169 169 F F F 29 580-599 96 96 96 167 167 167 F F F Max gen: 97, Max flush: 169 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:45.924288546Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:45.966985763Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:45.967400144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:45.998674669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:45.999374079Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:45.999408883Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:45.999423144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:45.999437296Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582} {"msg":"Crucible 23f7ffb7-5b15-448d-b94a-99f31f42262e has session id: 1346fb88-91fb-42ca-964d-7785beee199b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:45.999993802Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Upstairs opts: Upstairs UUID: 23f7ffb7-5b15-448d-b94a-99f31f42262e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000032346Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Crucible stats registered with UUID: 23f7ffb7-5b15-448d-b94a-99f31f42262e","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000056518Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:46.00007724Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000487602Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000519395Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582} {"msg":"23f7ffb7-5b15-448d-b94a-99f31f42262e active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000601433Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000617805Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"0","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000639747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"0","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000656999Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"1","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.00067131Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"1","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000685001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"2","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000699533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"2","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000712694Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000726136Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000790412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42612)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000890082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46428)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000928316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49519)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.000956339Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.001797674Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"1","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.001825407Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"2","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.001841128Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"0","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.002459631Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"0","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.002499144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"2","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.002522337Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"1","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003119417Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.0031434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[0]R flush_numbers[0..12]: [169, 167, 164, 162, 160, 169, 161, 156, 165, 159, 168, 168]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003257341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[0]R generation[0..12]: [97, 96, 94, 92, 90, 97, 91, 88, 94, 89, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003275873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003290164Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[1]R flush_numbers[0..12]: [169, 167, 164, 162, 160, 169, 161, 156, 165, 159, 168, 168]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003304256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[1]R generation[0..12]: [97, 96, 94, 92, 90, 97, 91, 88, 94, 89, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003317967Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003332848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[2]R flush_numbers[0..12]: [169, 167, 164, 162, 160, 169, 161, 156, 165, 159, 168, 168]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.00334641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[2]R generation[0..12]: [97, 96, 94, 92, 90, 97, 91, 88, 94, 89, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003360051Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003373933Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003386734Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Generation requested: 98 >= found:98"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003408736Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003421517Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003434019Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"":"downstairs","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.00344688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003458521Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"0","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003471943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"1","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003485194Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"client":"2","":"downstairs","session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003498375Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003510366Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"23f7ffb7-5b15-448d-b94a-99f31f42262e is now active with session: 1346fb88-91fb-42ca-964d-7785beee199b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003522938Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.003534609Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582,"session_id":"1346fb88-91fb-42ca-964d-7785beee199b"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:46.00354668Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 467 len 24576 data: 12 11 10 10 12 9 02/30 Read block 61 len 40960 03/30 Write block 146 len 20480 data: 6 7 6 10 8 04/30 Read block 212 len 32768 05/30 Read block 416 len 16384 06/30 Read block 53 len 16384 07/30 Read block 38 len 4096 08/30 Flush 09/30 Write block 297 len 32768 data: 7 7 8 11 10 13 12 12 10/30 Read block 32 len 28672 11/30 Read block 210 len 28672 12/30 Read block 29 len 16384 13/30 Write block 459 len 16384 data: 7 6 9 8 14/30 Read block 374 len 28672 15/30 Read block 174 len 24576 16/30 Write block 514 len 32768 data: 12 11 11 9 10 12 10 10 17/30 Write block 222 len 12288 data: 13 13 13 18/30 Write block 454 len 12288 data: 8 6 7 19/30 Write block 464 len 12288 data: 10 11 12 20/30 Read block 382 len 28672 21/30 Read block 222 len 40960 22/30 Flush 23/30 Write block 372 len 12288 data: 9 7 7 24/30 Read block 19 len 8192 25/30 Write block 319 len 36864 data: 14 13 14 13 13 13 10 9 8 26/30 Read block 498 len 12288 27/30 Read block 294 len 4096 28/30 Write block 87 len 32768 data: 11 10 11 10 7 7 6 4 29/30 Read block 50 len 32768 30/30 Write block 556 len 40960 data: 8 9 8 9 6 5 5 5 6 8 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:46.025180682Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 169 169 169 F F F 1 020-039 96 96 96 167 167 167 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 90 90 90 160 160 160 T T T 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 96 96 96 168 168 168 F F F 11 220-239 98 98 98 171 171 171 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 98 98 98 171 171 171 T T T 16 320-339 94 94 94 165 165 165 T T T 17 340-359 92 92 92 162 162 162 F F F 18 360-379 97 97 97 169 169 169 T T T 19 380-399 97 97 97 169 169 169 F F F 20 400-419 95 95 95 166 166 166 F F F 21 420-439 97 97 97 169 169 169 F F F 22 440-459 98 98 98 171 171 171 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 92 92 92 162 162 162 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 98 98 98 171 171 171 F F F 27 540-559 96 96 96 168 168 168 T T T 28 560-579 97 97 97 169 169 169 T T T 29 580-599 96 96 96 167 167 167 F F F Max gen: 98, Max flush: 171 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 169 169 169 F F F 1 020-039 96 96 96 167 167 167 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 99 99 99 172 172 172 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 96 96 96 168 168 168 F F F 11 220-239 98 98 98 171 171 171 F F F 12 240-259 91 91 91 161 161 161 F F F 13 260-279 94 94 94 164 164 164 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 99 99 99 172 172 172 F F F 16 320-339 99 99 99 172 172 172 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 99 99 99 172 172 172 F F F 19 380-399 97 97 97 169 169 169 F F F 20 400-419 95 95 95 166 166 166 F F F 21 420-439 97 97 97 169 169 169 F F F 22 440-459 98 98 98 171 171 171 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 92 92 92 162 162 162 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 98 98 98 171 171 171 F F F 27 540-559 99 99 99 172 172 172 F F F 28 560-579 99 99 99 172 172 172 F F F 29 580-599 96 96 96 167 167 167 F F F Max gen: 99, Max flush: 172 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.30369683Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.348133042Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.348844114Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.348886268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.3489008Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.348915031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590} {"msg":"Crucible cd6aab0f-1573-4c06-8539-53e19729338a has session id: 77135d84-a22d-4665-8022-cceec436b218","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.349544525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Upstairs opts: Upstairs UUID: cd6aab0f-1573-4c06-8539-53e19729338a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.349582718Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Crucible stats registered with UUID: cd6aab0f-1573-4c06-8539-53e19729338a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.3496028Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:50.349625213Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350098621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350130234Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590} {"msg":"cd6aab0f-1573-4c06-8539-53e19729338a active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350178178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350201641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"0","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350223493Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"0","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350244415Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"1","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350270468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"1","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.35029563Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350323113Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350348106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350374438Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350400951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"ds_connection connected from Ok(127.0.0.1:41966)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350427444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:50.350457467Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:50.35048574Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"ds_connection connected from Ok(127.0.0.1:57558)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350513102Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:31:50.350585119Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.350696181Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.351021034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"0","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.351059567Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"1","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.351686441Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"0","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.351817214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"1","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.35227864Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:31:50.352354428Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.352328364Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"ds_connection connected from Ok(127.0.0.1:62408)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.352532335Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.353479921Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.353988292Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[0]R flush_numbers[0..12]: [169, 167, 164, 162, 172, 169, 161, 170, 165, 159, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354556489Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[0]R generation[0..12]: [97, 96, 94, 92, 99, 97, 91, 98, 94, 89, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354584722Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354603564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[1]R flush_numbers[0..12]: [169, 167, 164, 162, 172, 169, 161, 170, 165, 159, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354620736Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[1]R generation[0..12]: [97, 96, 94, 92, 99, 97, 91, 98, 94, 89, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354637167Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354654939Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[2]R flush_numbers[0..12]: [169, 167, 164, 162, 172, 169, 161, 170, 165, 159, 168, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354671941Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[2]R generation[0..12]: [97, 96, 94, 92, 99, 97, 91, 98, 94, 89, 96, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354688593Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354705944Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354721756Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Generation requested: 100 >= found:100","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-02-27T17:32:00.354744098Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.35476036Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354776311Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"":"downstairs","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354792753Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354807374Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"0","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354827647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"1","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354844318Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"client":"2","":"downstairs","session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.35486071Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354875861Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"cd6aab0f-1573-4c06-8539-53e19729338a is now active with session: 77135d84-a22d-4665-8022-cceec436b218","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354891403Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354905904Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590,"session_id":"77135d84-a22d-4665-8022-cceec436b218"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:00.354920596Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 554 len 32768 data: 7 7 9 10 9 10 7 6 02/30 Write block 548 len 24576 data: 13 13 12 10 9 9 03/30 Read block 205 len 28672 04/30 Read block 173 len 36864 05/30 Write block 487 len 12288 data: 6 8 7 06/30 Read block 292 len 24576 07/30 Write block 382 len 32768 data: 8 9 9 11 11 10 10 8 08/30 Read block 485 len 28672 09/30 Read block 566 len 24576 10/30 Flush 11/30 Write block 200 len 40960 data: 8 8 9 6 4 6 7 6 5 6 12/30 Read block 466 len 4096 13/30 Write block 11 len 36864 data: 8 9 12 12 14 13 11 9 10 14/30 Read block 208 len 36864 15/30 Write block 541 len 4096 data: 8 16/30 Write block 587 len 20480 data: 7 8 8 9 10 17/30 Write block 247 len 32768 data: 8 9 11 12 12 14 17 14 18/30 Read block 400 len 24576 19/30 Write block 259 len 20480 data: 9 9 9 8 6 20/30 Read block 49 len 28672 21/30 Read block 32 len 20480 22/30 Read block 512 len 28672 23/30 Write block 303 len 40960 data: 13 13 9 9 7 6 8 8 8 7 24/30 Write block 31 len 24576 data: 8 8 9 9 10 10 25/30 Read block 268 len 36864 26/30 Read block 234 len 8192 27/30 Read block 7 len 40960 28/30 Read block 96 len 40960 29/30 Read block 439 len 8192 30/30 Write block 88 len 24576 data: 11 12 11 8 8 7 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 10 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:00.37677779Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 169 169 169 T T T 1 020-039 96 96 96 167 167 167 T T T 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 99 99 99 172 172 172 T T T 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 96 96 96 168 168 168 T T T 11 220-239 98 98 98 171 171 171 F F F 12 240-259 91 91 91 161 161 161 T T T 13 260-279 94 94 94 164 164 164 T T T 14 280-299 98 98 98 171 171 171 F F F 15 300-319 99 99 99 172 172 172 T T T 16 320-339 99 99 99 172 172 172 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 99 99 99 172 172 172 F F F 19 380-399 100 100 100 173 173 173 F F F 20 400-419 95 95 95 166 166 166 F F F 21 420-439 97 97 97 169 169 169 F F F 22 440-459 98 98 98 171 171 171 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 100 100 100 173 173 173 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 98 98 98 171 171 171 F F F 27 540-559 100 100 100 173 173 173 T T T 28 560-579 100 100 100 173 173 173 F F F 29 580-599 96 96 96 167 167 167 T T T Max gen: 100, Max flush: 173 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 174 174 174 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 101 101 101 174 174 174 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 89 89 89 159 159 159 F F F 10 200-219 101 101 101 174 174 174 F F F 11 220-239 98 98 98 171 171 171 F F F 12 240-259 101 101 101 174 174 174 F F F 13 260-279 101 101 101 174 174 174 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 101 101 101 174 174 174 F F F 16 320-339 99 99 99 172 172 172 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 99 99 99 172 172 172 F F F 19 380-399 100 100 100 173 173 173 F F F 20 400-419 95 95 95 166 166 166 F F F 21 420-439 97 97 97 169 169 169 F F F 22 440-459 98 98 98 171 171 171 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 100 100 100 173 173 173 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 98 98 98 171 171 171 F F F 27 540-559 101 101 101 174 174 174 F F F 28 560-579 100 100 100 173 173 173 F F F 29 580-599 101 101 101 174 174 174 F F F Max gen: 101, Max flush: 174 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.6425174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.687014728Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.68773106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.687772295Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.687787746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.687802938Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598} {"msg":"Crucible 725f81bf-72e0-4c51-be7d-09708d0cef0d has session id: cacdeedd-1aea-4280-bea0-6ac7ef20d715","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.68842505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Upstairs opts: Upstairs UUID: 725f81bf-72e0-4c51-be7d-09708d0cef0d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.688455893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Crucible stats registered with UUID: 725f81bf-72e0-4c51-be7d-09708d0cef0d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.688471965Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:04.688484846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.68901427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689042603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598} {"msg":"725f81bf-72e0-4c51-be7d-09708d0cef0d active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689108629Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689126181Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"0","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689149543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"0","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689164005Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689179746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689203059Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"2","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689229021Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"2","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689252524Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689279006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689303929Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"ds_connection connected from Ok(127.0.0.1:45433)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689353274Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:04.689392628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:04.689426441Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:04.689482567Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"ds_connection connected from Ok(127.0.0.1:51967)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689603999Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.689637132Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.690167436Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"0","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.69020671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"2","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.690920512Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"0","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.690948735Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"2","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.691554106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:04.691579488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.690909671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"ds_connection connected from Ok(127.0.0.1:58764)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.691062166Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.692186599Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.692715873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[0]R flush_numbers[0..12]: [174, 174, 164, 162, 174, 169, 161, 170, 165, 159, 174, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693341146Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[0]R generation[0..12]: [101, 101, 94, 92, 101, 97, 91, 98, 94, 89, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693366278Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.69338618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[1]R flush_numbers[0..12]: [174, 174, 164, 162, 174, 169, 161, 170, 165, 159, 174, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693401112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[1]R generation[0..12]: [101, 101, 94, 92, 101, 97, 91, 98, 94, 89, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693415043Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693428835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[2]R flush_numbers[0..12]: [174, 174, 164, 162, 174, 169, 161, 170, 165, 159, 174, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693442316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[2]R generation[0..12]: [101, 101, 94, 92, 101, 97, 91, 98, 94, 89, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693455358Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-02-27T17:32:14.693468939Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693489821Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693513723Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693538326Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693553587Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"":"downstairs","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693566409Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.69358071Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"0","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693604913Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"1","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693632595Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"client":"2","":"downstairs","session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693657898Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.69368064Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"725f81bf-72e0-4c51-be7d-09708d0cef0d is now active with session: cacdeedd-1aea-4280-bea0-6ac7ef20d715","v":0Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693707713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693729655Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598,"session_id":"cacdeedd-1aea-4280-bea0-6ac7ef20d715"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:14.693743247Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 92 len 32768 data: 9 8 5 2 2 2 3 3 02/30 Write block 295 len 4096 data: 6 03/30 Read block 480 len 20480 04/30 Write block 182 len 4096 data: 6 05/30 Flush 06/30 Write block 426 len 28672 data: 10 9 9 8 9 8 5 07/30 Read block 327 len 32768 08/30 Read block 204 len 28672 09/30 Read block 56 len 16384 10/30 Read block 395 len 40960 11/30 Write block 12 len 4096 data: 10 12/30 Write block 565 len 24576 data: 9 7 7 6 7 7 13/30 Write block 417 len 28672 data: 9 10 9 8 8 9 7 14/30 Write block 494 len 4096 data: 4 15/30 Read block 472 len 20480 16/30 Flush 17/30 Write block 431 len 16384 data: 9 6 4 5 18/30 Read block 16 len 36864 19/30 Write block 446 len 32768 data: 6 5 5 4 4 5 9 9 20/30 Read block 309 len 16384 21/30 Write block 364 len 32768 data: 8 8 8 9 8 10 8 10 22/30 Read block 220 len 16384 23/30 Read block 295 len 20480 24/30 Read block 321 len 28672 25/30 Flush 26/30 Write block 321 len 40960 data: 15 14 14 14 11 10 9 7 7 8 27/30 Read block 9 len 32768 28/30 Read block 161 len 4096 29/30 Read block 414 len 20480 30/30 Read block 456 len 12288 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:14.714608611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 102 102 102 175 175 175 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 102 102 102 175 175 175 F F F 10 200-219 101 101 101 174 174 174 F F F 11 220-239 98 98 98 171 171 171 F F F 12 240-259 101 101 101 174 174 174 F F F 13 260-279 101 101 101 174 174 174 F F F 14 280-299 102 102 102 175 175 175 F F F 15 300-319 101 101 101 174 174 174 F F F 16 320-339 99 99 99 172 172 172 T T T 17 340-359 92 92 92 162 162 162 F F F 18 360-379 102 102 102 177 177 177 F F F 19 380-399 100 100 100 173 173 173 F F F 20 400-419 102 102 102 176 176 176 F F F 21 420-439 102 102 102 177 177 177 F F F 22 440-459 102 102 102 177 177 177 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 102 102 102 176 176 176 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 98 98 98 171 171 171 F F F 27 540-559 101 101 101 174 174 174 F F F 28 560-579 102 102 102 176 176 176 F F F 29 580-599 101 101 101 174 174 174 F F F Max gen: 102, Max flush: 177 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 102 102 102 175 175 175 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 94 94 94 165 165 165 F F F 9 180-199 102 102 102 175 175 175 F F F 10 200-219 101 101 101 174 174 174 F F F 11 220-239 98 98 98 171 171 171 F F F 12 240-259 101 101 101 174 174 174 F F F 13 260-279 101 101 101 174 174 174 F F F 14 280-299 102 102 102 175 175 175 F F F 15 300-319 101 101 101 174 174 174 F F F 16 320-339 103 103 103 178 178 178 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 102 102 102 177 177 177 F F F 19 380-399 100 100 100 173 173 173 F F F 20 400-419 102 102 102 176 176 176 F F F 21 420-439 102 102 102 177 177 177 F F F 22 440-459 102 102 102 177 177 177 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 102 102 102 176 176 176 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 98 98 98 171 171 171 F F F 27 540-559 101 101 101 174 174 174 F F F 28 560-579 102 102 102 176 176 176 F F F 29 580-599 101 101 101 174 174 174 F F F Max gen: 103, Max flush: 178 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:18.934410077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:18.977478461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:18.977916355Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.009484799Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.010173058Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.010215293Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.010232284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.010251016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606} {"msg":"Crucible 522289d5-c709-4a25-8a10-a0f59105cc09 has session id: 716ffce4-49ef-4c92-b69b-90c0c21f1d42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.010841766Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Upstairs opts: Upstairs UUID: 522289d5-c709-4a25-8a10-a0f59105cc09, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.01087935Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Crucible stats registered with UUID: 522289d5-c709-4a25-8a10-a0f59105cc09","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.010895651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:19.010908283Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011403493Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011437726Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606} {"msg":"522289d5-c709-4a25-8a10-a0f59105cc09 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011494612Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011523074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"0","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011546497Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"0","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011564919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"1","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.01157994Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"1","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011593222Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"2","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011613154Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"2","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011635226Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011658368Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.01168041Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:53082)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011704023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:42029)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011728965Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"ds_connection connected from Ok(127.0.0.1:51046)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.011752978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.012605024Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"1","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.012634337Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"2","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.012878001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"0","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.013420386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"1","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.013451499Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"2","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.013467731Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"0","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014021577Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014121647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[0]R flush_numbers[0..12]: [176, 174, 164, 162, 175, 169, 161, 170, 165, 175, 174, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014197394Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[0]R generation[0..12]: [102, 101, 94, 92, 102, 97, 91, 98, 94, 102, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014215656Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014229818Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[1]R flush_numbers[0..12]: [176, 174, 164, 162, 175, 169, 161, 170, 165, 175, 174, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014243339Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[1]R generation[0..12]: [102, 101, 94, 92, 102, 97, 91, 98, 94, 102, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.0142567Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014270602Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[2]R flush_numbers[0..12]: [176, 174, 164, 162, 175, 169, 161, 170, 165, 175, 174, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014284233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[2]R generation[0..12]: [102, 101, 94, 92, 102, 97, 91, 98, 94, 102, 101, 98]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014297654Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014311326Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-02-27T17:32:19.014333508Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014346479Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014358871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014371432Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"":"downstairs","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014385403Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014397244Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"0","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014410076Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"1","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014422817Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"client":"2","":"downstairs","session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014435438Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.0144473Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"522289d5-c709-4a25-8a10-a0f59105cc09 is now active with session: 716ffce4-49ef-4c92-b69b-90c0c21f1d42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014459781Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014471412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606,"session_id":"716ffce4-49ef-4c92-b69b-90c0c21f1d42"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:19.014483463Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 542 len 12288 data: 9 9 9 02/30 Flush 03/30 Write block 525 len 32768 data: 9 11 12 12 13 10 10 6 04/30 Read block 111 len 24576 05/30 Write block 368 len 28672 data: 9 11 9 11 10 8 8 06/30 Write block 248 len 24576 data: 10 12 13 13 15 18 07/30 Read block 200 len 12288 08/30 Flush 09/30 Write block 179 len 28672 data: 6 6 6 7 5 8 9 10/30 Write block 270 len 40960 data: 7 5 6 5 5 4 5 5 5 5 11/30 Read block 432 len 20480 12/30 Write block 457 len 8192 data: 8 8 13/30 Read block 285 len 28672 14/30 Read block 272 len 24576 15/30 Read block 347 len 4096 16/30 Read block 241 len 24576 17/30 Flush 18/30 Write block 575 len 24576 data: 9 7 7 8 7 6 19/30 Read block 166 len 4096 20/30 Read block 332 len 16384 21/30 Read block 107 len 40960 22/30 Write block 245 len 12288 data: 8 8 9 23/30 Write block 199 len 28672 data: 10 9 9 10 7 5 7 24/30 Read block 529 len 40960 25/30 Read block 388 len 16384 26/30 Read block 524 len 28672 27/30 Read block 45 len 12288 28/30 Read block 175 len 32768 29/30 Read block 176 len 28672 30/30 Write block 232 len 12288 data: 7 8 7 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:19.035616115Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 102 102 102 175 175 175 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 104 104 104 181 181 181 F F F 9 180-199 104 104 104 181 181 181 T T T 10 200-219 101 101 101 174 174 174 T T T 11 220-239 98 98 98 171 171 171 T T T 12 240-259 104 104 104 180 180 180 T T T 13 260-279 104 104 104 181 181 181 F F F 14 280-299 102 102 102 175 175 175 F F F 15 300-319 101 101 101 174 174 174 F F F 16 320-339 103 103 103 178 178 178 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 100 100 100 173 173 173 F F F 20 400-419 102 102 102 176 176 176 F F F 21 420-439 102 102 102 177 177 177 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 102 102 102 176 176 176 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 104 104 104 180 180 180 F F F 27 540-559 104 104 104 179 179 179 F F F 28 560-579 102 102 102 176 176 176 T T T 29 580-599 101 101 101 174 174 174 T T T Max gen: 104, Max flush: 181 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 F F F 4 080-099 102 102 102 175 175 175 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 104 104 104 181 181 181 F F F 9 180-199 105 105 105 182 182 182 F F F 10 200-219 105 105 105 182 182 182 F F F 11 220-239 105 105 105 182 182 182 F F F 12 240-259 105 105 105 182 182 182 F F F 13 260-279 104 104 104 181 181 181 F F F 14 280-299 102 102 102 175 175 175 F F F 15 300-319 101 101 101 174 174 174 F F F 16 320-339 103 103 103 178 178 178 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 100 100 100 173 173 173 F F F 20 400-419 102 102 102 176 176 176 F F F 21 420-439 102 102 102 177 177 177 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 102 102 102 176 176 176 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 104 104 104 180 180 180 F F F 27 540-559 104 104 104 179 179 179 F F F 28 560-579 105 105 105 182 182 182 F F F 29 580-599 105 105 105 182 182 182 F F F Max gen: 105, Max flush: 182 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.312201558Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.356401746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.357143821Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.357188785Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.357204157Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.357218519Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614} {"msg":"Crucible 5f06cd62-fdf3-4128-962b-3c7efb152bc3 has session id: eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.357874125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Upstairs opts: Upstairs UUID: 5f06cd62-fdf3-4128-962b-3c7efb152bc3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.357914849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Crucible stats registered with UUID: 5f06cd62-fdf3-4128-962b-3c7efb152bc3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.357944162Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:23.357966924Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358348072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358381256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614} {"msg":"5f06cd62-fdf3-4128-962b-3c7efb152bc3 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358436231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358462124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"0","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358486616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"0","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358500908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358517959Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358542222Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"2","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358568644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"2","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358593377Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.35861882Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358641882Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"ds_connection connected from Ok(127.0.0.1:57901)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358664394Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"ds_connection connected from Ok(127.0.0.1:52804)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358688327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:23.35872146Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:23.358748993Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:23.358775545Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.358943072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.359437182Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"0","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.359463865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"2","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.360042833Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"2","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.360169106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"0","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.360597909Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:23.36070668Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.359890118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"ds_connection connected from Ok(127.0.0.1:39140)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.359954624Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.361184978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.361739584Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[0]R flush_numbers[0..12]: [176, 174, 164, 162, 175, 169, 161, 170, 181, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362337315Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[0]R generation[0..12]: [102, 101, 94, 92, 102, 97, 91, 98, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362363137Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362379869Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[1]R flush_numbers[0..12]: [176, 174, 164, 162, 175, 169, 161, 170, 181, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362394151Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[1]R generation[0..12]: [102, 101, 94, 92, 102, 97, 91, 98, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362426314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362441465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[2]R flush_numbers[0..12]: [176, 174, 164, 162, 175, 169, 161, 170, 181, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362455286Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[2]R generation[0..12]: [102, 101, 94, 92, 102, 97, 91, 98, 104, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362468868Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362482659Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362501031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362514663Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362527254Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362539725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"":"downstairs","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362555837Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362568138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"0","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362581479Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"1","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.3625942Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"client":"2","":"downstairs","session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362607422Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362619393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"5f06cd62-fdf3-4128-962b-3c7efb152bc3 is now active with session: eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362632004Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362643536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614,"session_id":"eeb2dcb0-5671-4ca2-b03f-12ee4c56ae10"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:33.362655327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 548 len 12288 data: 14 14 13 03/30 Read block 32 len 8192 04/30 Read block 14 len 20480 05/30 Flush 06/30 Write block 67 len 32768 data: 6 7 7 7 8 6 7 8 07/30 Write block 288 len 4096 data: 8 08/30 Read block 567 len 16384 09/30 Read block 171 len 16384 10/30 Write block 423 len 4096 data: 8 11/30 Read block 532 len 32768 12/30 Read block 107 len 4096 13/30 Read block 478 len 28672 14/30 Write block 167 len 4096 data: 6 15/30 Read block 254 len 4096 16/30 Write block 278 len 20480 data: 6 6 4 4 3 17/30 Write block 525 len 40960 data: 10 12 13 13 14 11 11 7 6 5 18/30 Write block 319 len 12288 data: 15 14 16 19/30 Read block 340 len 28672 20/30 Read block 477 len 8192 21/30 Read block 312 len 28672 22/30 Write block 268 len 40960 data: 10 8 8 6 7 6 6 5 6 6 23/30 Read block 542 len 28672 24/30 Read block 348 len 40960 25/30 Read block 512 len 40960 26/30 Write block 82 len 16384 data: 7 9 9 10 27/30 Read block 275 len 28672 28/30 Read block 484 len 36864 29/30 Read block 314 len 12288 30/30 Read block 341 len 12288 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 8 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 25 24 25 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:33.384073167Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 92 92 92 162 162 162 T T T 4 080-099 102 102 102 175 175 175 T T T 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 104 104 104 181 181 181 T T T 9 180-199 105 105 105 182 182 182 F F F 10 200-219 105 105 105 182 182 182 F F F 11 220-239 105 105 105 182 182 182 F F F 12 240-259 105 105 105 182 182 182 F F F 13 260-279 104 104 104 181 181 181 T T T 14 280-299 102 102 102 175 175 175 T T T 15 300-319 101 101 101 174 174 174 T T T 16 320-339 103 103 103 178 178 178 T T T 17 340-359 92 92 92 162 162 162 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 100 100 100 173 173 173 F F F 20 400-419 102 102 102 176 176 176 F F F 21 420-439 102 102 102 177 177 177 T T T 22 440-459 104 104 104 181 181 181 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 102 102 102 176 176 176 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 104 104 104 180 180 180 T T T 27 540-559 106 106 106 184 184 184 F F F 28 560-579 105 105 105 182 182 182 F F F 29 580-599 105 105 105 182 182 182 F F F Max gen: 106, Max flush: 184 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 107 107 107 185 185 185 F F F 4 080-099 107 107 107 185 185 185 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 98 98 98 170 170 170 F F F 8 160-179 107 107 107 185 185 185 F F F 9 180-199 105 105 105 182 182 182 F F F 10 200-219 105 105 105 182 182 182 F F F 11 220-239 105 105 105 182 182 182 F F F 12 240-259 105 105 105 182 182 182 F F F 13 260-279 107 107 107 185 185 185 F F F 14 280-299 107 107 107 185 185 185 F F F 15 300-319 107 107 107 185 185 185 F F F 16 320-339 107 107 107 185 185 185 F F F 17 340-359 92 92 92 162 162 162 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 100 100 100 173 173 173 F F F 20 400-419 102 102 102 176 176 176 F F F 21 420-439 107 107 107 185 185 185 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 102 102 102 176 176 176 F F F 25 500-519 98 98 98 171 171 171 F F F 26 520-539 107 107 107 185 185 185 F F F 27 540-559 106 106 106 184 184 184 F F F 28 560-579 105 105 105 182 182 182 F F F 29 580-599 105 105 105 182 182 182 F F F Max gen: 107, Max flush: 185 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.646720665Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.692399182Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.693161139Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.693201643Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.693216914Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.693230966Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622} {"msg":"Crucible 76e38f6e-0644-40c7-8f1c-e341a14ead55 has session id: 3973ce3a-7546-4b10-9b2f-34bfaa155c02","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.693833306Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Upstairs opts: Upstairs UUID: 76e38f6e-0644-40c7-8f1c-e341a14ead55, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.69387172Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Crucible stats registered with UUID: 76e38f6e-0644-40c7-8f1c-e341a14ead55","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.693896083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:47.693920845Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694408215Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694434147Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622} {"msg":"76e38f6e-0644-40c7-8f1c-e341a14ead55 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694505804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694524106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"0","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694549579Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"0","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.6945641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"1","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694578081Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"1","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694591023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"2","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694605034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"2","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694618086Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694641058Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.6946639Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"ds_connection connected from Ok(127.0.0.1:35619)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694796364Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"ds_connection connected from Ok(127.0.0.1:45761)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.694834887Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"ds_connection connected from Ok(127.0.0.1:34683)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.69486178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.69565323Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"2","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.695691484Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"0","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.696335339Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"0","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.696371673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"2","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.696462602Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"1","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.696990495Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697033419Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"1","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697050861Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"[0]R flush_numbers[0..12]: [176, 174, 164, 185, 185, 169, 161, 170, 185, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697598206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"[0]R generation[0..12]: [102, 101, 94, 107, 107, 97, 91, 98, 107, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.69763348Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697660012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"[1]R flush_numbers[0..12]: [176, 174, 164, 185, 185, 169, 161, 170, 185, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697686195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"[1]R generation[0..12]: [102, 101, 94, 107, 107, 97, 91, 98, 107, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697710608Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.69773832Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"[2]R flush_numbers[0..12]: [176, 174, 164, 185, 185, 169, 161, 170, 185, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697765883Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"}Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [102, 101, 94, 107, 107, 97, 91, 98, 107, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697799866Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697828509Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697854232Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697879785Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697905777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.69793134Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"":"downstairs","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"No downstairs reconciliation required","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2025-02-27T17:32:47.697960513Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.697985065Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"0","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.698010868Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"1","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.69802822Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"client":"2","":"downstairs","session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.698041481Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.698053692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"76e38f6e-0644-40c7-8f1c-e341a14ead55 is now active with session: 3973ce3a-7546-4b10-9b2f-34bfaa155c02","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.698066303Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.698081755Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622,"session_id":"3973ce3a-7546-4b10-9b2f-34bfaa155c02"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:32:47.698094016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 211 len 12288 data: 7 8 9 02/30 Read block 380 len 4096 03/30 Read block 287 len 4096 04/30 Write block 276 len 28672 data: 7 7 7 7 5 5 4 05/30 Write block 149 len 20480 data: 11 9 10 9 9 06/30 Read block 572 len 12288 07/30 Write block 306 len 12288 data: 10 8 7 08/30 Write block 518 len 28672 data: 11 13 11 11 10 8 9 09/30 Write block 87 len 40960 data: 12 12 13 12 9 10 9 6 3 3 10/30 Write block 228 len 32768 data: 9 7 9 9 8 9 8 10 11/30 Write block 299 len 16384 data: 9 12 11 14 12/30 Write block 399 len 24576 data: 6 6 6 6 5 7 13/30 Write block 569 len 36864 data: 8 8 7 10 9 8 10 8 8 14/30 Read block 448 len 4096 15/30 Write block 208 len 28672 data: 6 7 7 8 9 10 9 16/30 Read block 568 len 12288 17/30 Flush 18/30 Write block 62 len 36864 data: 8 8 7 7 6 7 8 8 8 19/30 Write block 312 len 20480 data: 8 6 6 8 8 20/30 Read block 563 len 40960 21/30 Read block 100 len 4096 22/30 Write block 321 len 12288 data: 17 15 15 23/30 Read block 237 len 12288 24/30 Read block 422 len 4096 25/30 Write block 185 len 8192 data: 10 8 26/30 Read block 482 len 12288 27/30 Write block 343 len 32768 data: 7 3 2 2 4 6 6 7 28/30 Write block 141 len 20480 data: 15 13 12 7 7 29/30 Write block 76 len 16384 data: 10 7 8 10 30/30 Read block 497 len 12288 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 9 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:32:47.716629036Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 107 107 107 185 185 185 T T T 4 080-099 108 108 108 186 186 186 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 108 108 108 186 186 186 T T T 8 160-179 107 107 107 185 185 185 F F F 9 180-199 105 105 105 182 182 182 T T T 10 200-219 108 108 108 186 186 186 F F F 11 220-239 108 108 108 186 186 186 F F F 12 240-259 105 105 105 182 182 182 F F F 13 260-279 108 108 108 186 186 186 F F F 14 280-299 108 108 108 186 186 186 F F F 15 300-319 108 108 108 186 186 186 T T T 16 320-339 107 107 107 185 185 185 T T T 17 340-359 92 92 92 162 162 162 T T T 18 360-379 104 104 104 180 180 180 F F F 19 380-399 108 108 108 186 186 186 F F F 20 400-419 108 108 108 186 186 186 F F F 21 420-439 107 107 107 185 185 185 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 102 102 102 176 176 176 F F F 25 500-519 108 108 108 186 186 186 F F F 26 520-539 108 108 108 186 186 186 F F F 27 540-559 106 106 106 184 184 184 F F F 28 560-579 108 108 108 186 186 186 F F F 29 580-599 105 105 105 182 182 182 F F F Max gen: 108, Max flush: 186 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 176 176 176 F F F 1 020-039 101 101 101 174 174 174 F F F 2 040-059 94 94 94 164 164 164 F F F 3 060-079 109 109 109 187 187 187 F F F 4 080-099 108 108 108 186 186 186 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 109 109 109 187 187 187 F F F 8 160-179 107 107 107 185 185 185 F F F 9 180-199 109 109 109 187 187 187 F F F 10 200-219 108 108 108 186 186 186 F F F 11 220-239 108 108 108 186 186 186 F F F 12 240-259 105 105 105 182 182 182 F F F 13 260-279 108 108 108 186 186 186 F F F 14 280-299 108 108 108 186 186 186 F F F 15 300-319 109 109 109 187 187 187 F F F 16 320-339 109 109 109 187 187 187 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 108 108 108 186 186 186 F F F 20 400-419 108 108 108 186 186 186 F F F 21 420-439 107 107 107 185 185 185 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 98 98 98 171 171 171 F F F 24 480-499 102 102 102 176 176 176 F F F 25 500-519 108 108 108 186 186 186 F F F 26 520-539 108 108 108 186 186 186 F F F 27 540-559 106 106 106 184 184 184 F F F 28 560-579 108 108 108 186 186 186 F F F 29 580-599 105 105 105 182 182 182 F F F Max gen: 109, Max flush: 187 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:01.972590006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.01922545Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.020094417Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.020152813Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.020177416Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.020195597Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630} {"msg":"Crucible 1dfbe52c-2fcd-4448-8051-59941edb4ffb has session id: 2f6749c6-aa2e-4431-9e1a-b133f9cd7010","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.020623631Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Upstairs opts: Upstairs UUID: 1dfbe52c-2fcd-4448-8051-59941edb4ffb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.020669865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Crucible stats registered with UUID: 1dfbe52c-2fcd-4448-8051-59941edb4ffb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.020700128Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:02.020728551Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021206079Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021232752Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630} {"msg":"1dfbe52c-2fcd-4448-8051-59941edb4ffb active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021295268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.02131314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"0","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021337083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"0","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021359065Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"1","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021386457Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"1","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.02140758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"2","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021437033Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"2","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021463405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021490888Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021715821Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"ds_connection connected from Ok(127.0.0.1:33244)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021744144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"ds_connection connected from Ok(127.0.0.1:60704)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021764046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"ds_connection connected from Ok(127.0.0.1:56350)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.021789488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.022333613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"2","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.022359726Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"0","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.02279495Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"1","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.023060226Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"0","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.023307701Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"2","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.023454586Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"1","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.02369444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.023782389Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[0]R flush_numbers[0..12]: [176, 174, 164, 187, 186, 169, 161, 187, 185, 187, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024027564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[0]R generation[0..12]: [102, 101, 94, 109, 108, 97, 91, 109, 107, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024050936Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024065978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[1]R flush_numbers[0..12]: [176, 174, 164, 187, 186, 169, 161, 187, 185, 187, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024080169Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[1]R generation[0..12]: [102, 101, 94, 109, 108, 97, 91, 109, 107, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024093971Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024107892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[2]R flush_numbers[0..12]: [176, 174, 164, 187, 186, 169, 161, 187, 185, 187, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024121764Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[2]R generation[0..12]: [102, 101, 94, 109, 108, 97, 91, 109, 107, 109, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024136035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024149786Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Max found gen is 110","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-02-27T17:33:02.024168768Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024181959Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024194621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024207252Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"":"downstairs","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024223344Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024235455Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"0","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024248426Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"1","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:33:02.024261187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"client":"2","":"downstairs","session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024274279Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.02428622Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"1dfbe52c-2fcd-4448-8051-59941edb4ffb is now active with session: 2f6749c6-aa2e-4431-9e1a-b133f9cd7010","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024298901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024310732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630,"session_id":"2f6749c6-aa2e-4431-9e1a-b133f9cd7010"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:02.024322744Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 71 len 12288 data: 9 7 8 02/30 Read block 62 len 16384 03/30 Read block 390 len 32768 04/30 Write block 490 len 36864 data: 5 4 4 4 5 5 6 6 7 05/30 Write block 215 len 24576 data: 11 11 12 12 12 11 06/30 Read block 477 len 28672 07/30 Flush 08/30 Write block 538 len 12288 data: 7 6 6 09/30 Read block 594 len 12288 10/30 Write block 2 len 28672 data: 5 9 10 10 11 11 13 11/30 Read block 518 len 40960 12/30 Write block 209 len 36864 data: 8 8 9 10 11 10 12 12 13 13/30 Write block 305 len 8192 data: 10 11 14/30 Read block 444 len 36864 15/30 Write block 462 len 40960 data: 9 7 11 12 13 13 12 11 11 13 16/30 Write block 287 len 4096 data: 9 17/30 Write block 31 len 16384 data: 9 9 10 10 18/30 Write block 242 len 4096 data: 4 19/30 Read block 167 len 8192 20/30 Flush 21/30 Write block 249 len 12288 data: 13 14 14 22/30 Flush 23/30 Write block 484 len 20480 data: 6 6 6 7 9 24/30 Write block 564 len 40960 data: 7 10 8 8 7 9 9 8 11 10 25/30 Write block 329 len 4096 data: 8 26/30 Read block 247 len 12288 27/30 Write block 431 len 32768 data: 10 7 5 6 5 7 6 8 28/30 Read block 320 len 24576 29/30 Read block 135 len 4096 30/30 Write block 51 len 28672 data: 7 8 6 10 12 11 12 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:02.043688587Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 94 94 94 164 164 164 T T T 3 060-079 110 110 110 188 188 188 F F F 4 080-099 108 108 108 186 186 186 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 109 109 109 187 187 187 F F F 8 160-179 107 107 107 185 185 185 F F F 9 180-199 109 109 109 187 187 187 F F F 10 200-219 110 110 110 189 189 189 F F F 11 220-239 110 110 110 188 188 188 F F F 12 240-259 110 110 110 190 190 190 F F F 13 260-279 108 108 108 186 186 186 F F F 14 280-299 110 110 110 189 189 189 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 109 109 109 187 187 187 T T T 17 340-359 109 109 109 187 187 187 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 108 108 108 186 186 186 F F F 20 400-419 108 108 108 186 186 186 F F F 21 420-439 107 107 107 185 185 185 T T T 22 440-459 104 104 104 181 181 181 F F F 23 460-479 110 110 110 189 189 189 F F F 24 480-499 110 110 110 188 188 188 T T T 25 500-519 108 108 108 186 186 186 F F F 26 520-539 110 110 110 189 189 189 F F F 27 540-559 110 110 110 189 189 189 F F F 28 560-579 108 108 108 186 186 186 T T T 29 580-599 105 105 105 182 182 182 F F F Max gen: 110, Max flush: 190 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 110 110 110 189 189 189 F F F 2 040-059 111 111 111 191 191 191 F F F 3 060-079 110 110 110 188 188 188 F F F 4 080-099 108 108 108 186 186 186 F F F 5 100-119 97 97 97 169 169 169 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 109 109 109 187 187 187 F F F 8 160-179 107 107 107 185 185 185 F F F 9 180-199 109 109 109 187 187 187 F F F 10 200-219 110 110 110 189 189 189 F F F 11 220-239 110 110 110 188 188 188 F F F 12 240-259 110 110 110 190 190 190 F F F 13 260-279 108 108 108 186 186 186 F F F 14 280-299 110 110 110 189 189 189 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 191 191 191 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 108 108 108 186 186 186 F F F 20 400-419 108 108 108 186 186 186 F F F 21 420-439 111 111 111 191 191 191 F F F 22 440-459 104 104 104 181 181 181 F F F 23 460-479 110 110 110 189 189 189 F F F 24 480-499 111 111 111 191 191 191 F F F 25 500-519 108 108 108 186 186 186 F F F 26 520-539 110 110 110 189 189 189 F F F 27 540-559 110 110 110 189 189 189 F F F 28 560-579 111 111 111 191 191 191 F F F 29 580-599 105 105 105 182 182 182 F F F Max gen: 111, Max flush: 191 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.289481955Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.33474458Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.335603157Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.335645681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.335660212Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.335674534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638} {"msg":"Crucible 6d610077-d141-4f68-b883-7a4caebe5e3f has session id: ba724db1-8ad3-451d-825e-9f4a8c189869","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.336287786Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Upstairs opts: Upstairs UUID: 6d610077-d141-4f68-b883-7a4caebe5e3f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.336322389Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Crucible stats registered with UUID: 6d610077-d141-4f68-b883-7a4caebe5e3f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.336337741Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:06.336350152Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.33682631Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.336855273Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638} {"msg":"6d610077-d141-4f68-b883-7a4caebe5e3f active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.33692182Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.336946562Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"0","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.336971415Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"0","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.336985616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"1","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.336999748Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"1","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337012659Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"2","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.33702619Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"2","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337049182Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337074465Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337099138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"ds_connection connected from Ok(127.0.0.1:42816)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337214319Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"ds_connection connected from Ok(127.0.0.1:36142)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337254483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"ds_connection connected from Ok(127.0.0.1:59573)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337279016Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337893418Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"2","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.337921141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"0","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.338545573Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"2","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.33871162Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"0","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.339151505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.339273757Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.339350865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"1","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.339886199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"1","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[0]R flush_numbers[0..12]: [189, 189, 191, 188, 186, 169, 161, 187, 185, 187, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340421173Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[0]R generation[0..12]: [110, 110, 111, 110, 108, 97, 91, 109, 107, 109, 110, 110]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340447345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340478048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[1]R flush_numbers[0..12]: [189, 189, 191, 188, 186, 169, 161, 187, 185, 187, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.34049355Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[1]R generation[0..12]: [110, 110, 111, 110, 108, 97, 91, 109, 107, 109, 110, 110]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340507381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340521143Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[2]R flush_numbers[0..12]: [189, 189, 191, 188, 186, 169, 161, 187, 185, 187, 189, 188]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340535254Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[2]R generation[0..12]: [110, 110, 111, 110, 108, 97, 91, 109, 107, 109, 110, 110]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340552166Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-02-27T17:33:06.340572498Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340586849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340600181Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340612862Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340625553Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"":"downstairs","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340639344Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340651176Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"0","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340664067Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"1","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340676908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"client":"2","":"downstairs","session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.34068977Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340701651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"6d610077-d141-4f68-b883-7a4caebe5e3f is now active with session: ba724db1-8ad3-451d-825e-9f4a8c189869","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340714092Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340725813Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638,"session_id":"ba724db1-8ad3-451d-825e-9f4a8c189869"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:06.340737665Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 468 len 40960 data: 13 12 12 14 10 7 6 6 6 4 02/30 Write block 150 len 4096 data: 10 03/30 Read block 407 len 4096 04/30 Write block 502 len 20480 data: 6 6 5 6 6 05/30 Write block 246 len 4096 data: 9 06/30 Write block 530 len 12288 data: 12 12 8 07/30 Write block 454 len 20480 data: 9 7 8 9 9 08/30 Read block 190 len 40960 09/30 Read block 405 len 24576 10/30 Flush 11/30 Flush 12/30 Write block 210 len 16384 data: 9 10 11 12 13/30 Flush 14/30 Write block 245 len 28672 data: 9 10 10 11 14 15 15 15/30 Flush 16/30 Write block 417 len 24576 data: 10 11 10 9 9 10 17/30 Read block 333 len 20480 18/30 Read block 290 len 24576 19/30 Read block 72 len 8192 20/30 Read block 513 len 4096 21/30 Write block 90 len 12288 data: 13 10 11 22/30 Read block 31 len 20480 23/30 Read block 1 len 16384 24/30 Read block 45 len 40960 25/30 Write block 109 len 20480 data: 10 11 9 10 8 26/30 Write block 252 len 36864 data: 16 19 15 10 10 9 8 10 10 27/30 Write block 488 len 16384 data: 10 8 6 5 28/30 Read block 119 len 20480 29/30 Write block 37 len 36864 data: 11 9 10 9 9 10 10 7 6 30/30 Read block 28 len 20480 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 15 15 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:06.361168405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 110 110 110 189 189 189 T T T 2 040-059 111 111 111 191 191 191 T T T 3 060-079 110 110 110 188 188 188 F F F 4 080-099 108 108 108 186 186 186 T T T 5 100-119 97 97 97 169 169 169 T T T 6 120-139 91 91 91 161 161 161 F F F 7 140-159 112 112 112 192 192 192 F F F 8 160-179 107 107 107 185 185 185 F F F 9 180-199 109 109 109 187 187 187 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 188 188 188 F F F 12 240-259 112 112 112 195 195 195 T T T 13 260-279 108 108 108 186 186 186 T T T 14 280-299 110 110 110 189 189 189 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 191 191 191 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 108 108 108 186 186 186 F F F 20 400-419 108 108 108 186 186 186 T T T 21 420-439 111 111 111 191 191 191 T T T 22 440-459 112 112 112 192 192 192 F F F 23 460-479 112 112 112 192 192 192 F F F 24 480-499 111 111 111 191 191 191 T T T 25 500-519 112 112 112 192 192 192 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 110 110 110 189 189 189 F F F 28 560-579 111 111 111 191 191 191 F F F 29 580-599 105 105 105 182 182 182 F F F Max gen: 112, Max flush: 195 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 110 110 110 188 188 188 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 112 112 112 192 192 192 F F F 8 160-179 107 107 107 185 185 185 F F F 9 180-199 109 109 109 187 187 187 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 110 110 110 188 188 188 F F F 12 240-259 113 113 113 196 196 196 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 110 110 110 189 189 189 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 191 191 191 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 104 104 104 180 180 180 F F F 19 380-399 108 108 108 186 186 186 F F F 20 400-419 113 113 113 196 196 196 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 112 112 112 192 192 192 F F F 23 460-479 112 112 112 192 192 192 F F F 24 480-499 113 113 113 196 196 196 F F F 25 500-519 112 112 112 192 192 192 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 110 110 110 189 189 189 F F F 28 560-579 111 111 111 191 191 191 F F F 29 580-599 105 105 105 182 182 182 F F F Max gen: 113, Max flush: 196 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.623778419Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.668168216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.668967307Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.669018862Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.669039134Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.669054156Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646} {"msg":"Crucible fd93ef53-c421-4269-9e56-00afebd40fee has session id: 27bdeaea-e769-45ef-8e13-cd9d66a2e695","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.669774848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Upstairs opts: Upstairs UUID: fd93ef53-c421-4269-9e56-00afebd40fee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.669818413Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Crucible stats registered with UUID: fd93ef53-c421-4269-9e56-00afebd40fee","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.669841325Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:20.669863357Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670360297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.67038409Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646} {"msg":"fd93ef53-c421-4269-9e56-00afebd40fee active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670467138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.67048426Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"0","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670510043Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"0","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670523414Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"1","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670537425Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"1","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670550376Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"2","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670564198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"2","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.67058608Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670620914Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670646136Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"ds_connection connected from Ok(127.0.0.1:43103)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670671049Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"ds_connection connected from Ok(127.0.0.1:34127)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.67077819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"ds_connection connected from Ok(127.0.0.1:59474)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.670970369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.671508203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"1","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.671546877Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"0","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.672182821Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"1","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.672210474Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"0","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.672806624Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.672894623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.673717026Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"2","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674239949Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"2","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"[0]R flush_numbers[0..12]: [189, 196, 196, 188, 196, 196, 161, 192, 185, 187, 194, 188]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674774922Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"[0]R generation[0..12]: [110, 113, 113, 110, 113, 113, 91, 112, 107, 109, 112, 110]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674803505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674825548Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"[1]R flush_numbers[0..12]: [189, 196, 196, 188, 196, 196, 161, 192, 185, 187, 194, 188]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674843229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"[1]R generation[0..12]: [110, 113, 113, 110, 113, 113, 91, 112, 107, 109, 112, 110]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674860051Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674877343Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"[2]R flush_numbers[0..12]: [189, 196, 196, 188, 196, 196, 161, 192, 185, 187, 194, 188]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674894225Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [110, 113, 113, 110, 113, 113, 91, 112, 107, 109, 112, 110]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674916737Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674934089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.67494987Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674966002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.674981623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675016397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"":"downstairs","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675035369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.67505029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"0","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675066352Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675089584Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"client":"2","":"downstairs","session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675105516Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675120287Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"fd93ef53-c421-4269-9e56-00afebd40fee is now active with session: 27bdeaea-e769-45ef-8e13-cd9d66a2e695","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675135919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675150411Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646,"session_id":"27bdeaea-e769-45ef-8e13-cd9d66a2e695"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:20.675165372Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 378 len 32768 data: 8 8 8 7 9 10 10 12 02/30 Write block 553 len 36864 data: 10 8 8 10 11 10 11 8 7 03/30 Write block 383 len 12288 data: 11 11 13 04/30 Write block 548 len 8192 data: 15 15 05/30 Write block 291 len 12288 data: 8 7 5 06/30 Write block 191 len 20480 data: 5 6 9 9 11 07/30 Write block 375 len 16384 data: 6 6 8 9 08/30 Read block 527 len 28672 09/30 Flush 10/30 Flush 11/30 Write block 401 len 36864 data: 7 7 6 8 7 7 7 12 12 12/30 Read block 454 len 16384 13/30 Write block 257 len 12288 data: 10 9 11 14/30 Read block 553 len 24576 15/30 Flush 16/30 Write block 223 len 12288 data: 14 14 14 17/30 Read block 470 len 20480 18/30 Write block 74 len 16384 data: 9 10 11 8 19/30 Flush 20/30 Write block 418 len 4096 data: 12 21/30 Write block 555 len 12288 data: 9 11 12 22/30 Read block 567 len 40960 23/30 Read block 504 len 32768 24/30 Write block 566 len 36864 data: 9 9 8 10 10 9 12 11 9 25/30 Read block 102 len 20480 26/30 Write block 549 len 40960 data: 16 14 11 10 11 9 10 12 13 11 27/30 Write block 140 len 32768 data: 15 16 14 13 8 8 7 8 28/30 Write block 594 len 20480 data: 7 4 3 3 4 29/30 Write block 550 len 8192 data: 15 12 30/30 Write block 384 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Sent Done Sent false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 3 4 11 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:20.692928584Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 200 200 200 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 112 112 112 192 192 192 T T T 8 160-179 107 107 107 185 185 185 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 114 114 114 200 200 200 F F F 12 240-259 114 114 114 199 199 199 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 191 191 191 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 114 114 114 197 197 197 F F F 19 380-399 114 114 114 197 197 197 T T T 20 400-419 114 114 114 199 199 199 T T T 21 420-439 113 113 113 196 196 196 F F F 22 440-459 112 112 112 192 192 192 F F F 23 460-479 112 112 112 192 192 192 F F F 24 480-499 113 113 113 196 196 196 F F F 25 500-519 112 112 112 192 192 192 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 114 114 114 197 197 197 T T T 28 560-579 114 114 114 197 197 197 T T T 29 580-599 105 105 105 182 182 182 T T T Max gen: 114, Max flush: 200 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 200 200 200 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 115 115 115 201 201 201 F F F 8 160-179 107 107 107 185 185 185 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 114 114 114 200 200 200 F F F 12 240-259 114 114 114 199 199 199 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 110 110 110 189 189 189 F F F 16 320-339 111 111 111 191 191 191 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 114 114 114 197 197 197 F F F 19 380-399 115 115 115 201 201 201 F F F 20 400-419 115 115 115 201 201 201 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 112 112 112 192 192 192 F F F 23 460-479 112 112 112 192 192 192 F F F 24 480-499 113 113 113 196 196 196 F F F 25 500-519 112 112 112 192 192 192 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 115 115 115 201 201 201 F F F 28 560-579 115 115 115 201 201 201 F F F 29 580-599 115 115 115 201 201 201 F F F Max gen: 115, Max flush: 201 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.944172561Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.988837896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.989704423Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.989746357Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.989761359Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.98977585Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654} {"msg":"Crucible e567f156-4db5-495f-a97c-4e43c54a4a42 has session id: ee33c311-8c70-427b-a98c-bf2d800858d4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.990410634Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Upstairs opts: Upstairs UUID: e567f156-4db5-495f-a97c-4e43c54a4a42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.990450098Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Crucible stats registered with UUID: e567f156-4db5-495f-a97c-4e43c54a4a42","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.990471861Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:24.990491003Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991079532Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991105534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654} {"msg":"e567f156-4db5-495f-a97c-4e43c54a4a42 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991178122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991195594Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"0","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991223466Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"0","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991237388Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991251429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991264341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"2","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991277932Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"2","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991290783Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991304614Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991317676Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:24.991546169Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:24.991609795Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41103)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991700414Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:46617)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991733028Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:24.991764991Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.991791003Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.992285464Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"0","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.992312776Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"2","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.992954961Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"2","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.993103466Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"0","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.993513807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:24.99364304Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.992784784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43410)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.992932999Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.99413544Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.994749422Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[0]R flush_numbers[0..12]: [189, 196, 196, 200, 196, 196, 161, 201, 185, 197, 194, 200]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995347692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[0]R generation[0..12]: [110, 113, 113, 114, 113, 113, 91, 115, 107, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995367024Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995381835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[1]R flush_numbers[0..12]: [189, 196, 196, 200, 196, 196, 161, 201, 185, 197, 194, 200]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995395197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[1]R generation[0..12]: [110, 113, 113, 114, 113, 113, 91, 115, 107, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995413039Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.99542716Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[2]R flush_numbers[0..12]: [189, 196, 196, 200, 196, 196, 161, 201, 185, 197, 194, 200]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995440581Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[2]R generation[0..12]: [110, 113, 113, 114, 113, 113, 91, 115, 107, 114, 112, 114]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995453653Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995467464Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995480085Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2025-02-27T17:33:34.995493117Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.99552617Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995539362Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"":"downstairs","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995552083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995563894Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"0","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995576785Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"1","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995589447Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"client":"2","":"downstairs","session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995602248Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995614039Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"e567f156-4db5-495f-a97c-4e43c54a4a42 is now active with session: ee33c311-8c70-427b-a98c-bf2d800858d4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.99562874Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995640582Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654,"session_id":"ee33c311-8c70-427b-a98c-bf2d800858d4"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:34.995652463Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 238 len 36864 data: 10 10 6 6 5 5 7 10 11 02/30 Flush 03/30 Flush 04/30 Write block 151 len 24576 data: 11 10 10 9 8 6 05/30 Write block 442 len 4096 data: 10 06/30 Write block 106 len 24576 data: 8 8 8 11 12 10 07/30 Write block 41 len 16384 data: 10 11 11 8 08/30 Read block 485 len 24576 09/30 Flush 10/30 Write block 301 len 24576 data: 12 15 14 14 11 12 11/30 Read block 20 len 8192 12/30 Write block 160 len 32768 data: 5 6 6 7 7 7 6 7 13/30 Write block 494 len 16384 data: 6 6 7 7 14/30 Flush 15/30 Write block 85 len 40960 data: 11 13 13 13 14 14 11 12 10 7 16/30 Read block 205 len 24576 17/30 Write block 290 len 16384 data: 7 9 8 6 18/30 Write block 316 len 16384 data: 9 12 13 16 19/30 Read block 569 len 32768 20/30 Read block 327 len 8192 21/30 Flush 22/30 Write block 76 len 28672 data: 12 9 9 11 10 8 8 23/30 Read block 512 len 24576 24/30 Read block 387 len 24576 25/30 Write block 58 len 4096 data: 11 26/30 Read block 527 len 20480 27/30 Read block 74 len 12288 28/30 Write block 308 len 32768 data: 8 9 9 9 9 7 7 9 29/30 Write block 371 len 16384 data: 12 11 9 9 30/30 Write block 226 len 24576 data: 11 9 10 8 10 10 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:35.015416536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 116 116 116 204 204 204 T T T 3 060-079 114 114 114 200 200 200 T T T 4 080-099 116 116 116 206 206 206 T T T 5 100-119 116 116 116 204 204 204 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 116 116 116 204 204 204 F F F 8 160-179 116 116 116 205 205 205 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 116 116 116 202 202 202 T T T 12 240-259 116 116 116 202 202 202 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 116 116 116 206 206 206 F F F 15 300-319 116 116 116 206 206 206 T T T 16 320-339 111 111 111 191 191 191 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 114 114 114 197 197 197 T T T 19 380-399 115 115 115 201 201 201 F F F 20 400-419 115 115 115 201 201 201 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 116 116 116 204 204 204 F F F 23 460-479 112 112 112 192 192 192 F F F 24 480-499 116 116 116 205 205 205 F F F 25 500-519 112 112 112 192 192 192 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 115 115 115 201 201 201 F F F 28 560-579 115 115 115 201 201 201 F F F 29 580-599 115 115 115 201 201 201 F F F Max gen: 116, Max flush: 206 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 117 117 117 207 207 207 F F F 4 080-099 117 117 117 207 207 207 F F F 5 100-119 116 116 116 204 204 204 F F F 6 120-139 91 91 91 161 161 161 F F F 7 140-159 116 116 116 204 204 204 F F F 8 160-179 116 116 116 205 205 205 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 117 117 117 207 207 207 F F F 12 240-259 116 116 116 202 202 202 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 116 116 116 206 206 206 F F F 15 300-319 117 117 117 207 207 207 F F F 16 320-339 111 111 111 191 191 191 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 117 117 117 207 207 207 F F F 19 380-399 115 115 115 201 201 201 F F F 20 400-419 115 115 115 201 201 201 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 116 116 116 204 204 204 F F F 23 460-479 112 112 112 192 192 192 F F F 24 480-499 116 116 116 205 205 205 F F F 25 500-519 112 112 112 192 192 192 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 115 115 115 201 201 201 F F F 28 560-579 115 115 115 201 201 201 F F F 29 580-599 115 115 115 201 201 201 F F F Max gen: 117, Max flush: 207 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.265593296Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.309935229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.310721828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.310771933Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.310794175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.310809077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662} {"msg":"Crucible c4117416-0398-4301-95f3-360c25e32b10 has session id: 03dc8ca8-c757-4063-9d77-7f3306c21078","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.311322918Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Upstairs opts: Upstairs UUID: c4117416-0398-4301-95f3-360c25e32b10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.311361662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Crucible stats registered with UUID: c4117416-0398-4301-95f3-360c25e32b10","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.311385115Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:49.311398536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.311911988Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.311985665Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662} {"msg":"c4117416-0398-4301-95f3-360c25e32b10 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312022689Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.31203612Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"0","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312059033Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"0","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312072994Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"1","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312086706Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"1","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312099537Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"2","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312118329Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"2","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.31213168Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312145251Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312158443Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"ds_connection connected from Ok(127.0.0.1:59324)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312356833Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"ds_connection connected from Ok(127.0.0.1:49255)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.312397937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"ds_connection connected from Ok(127.0.0.1:34872)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.31242428Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.313071365Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"1","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.313098077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"0","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.313759384Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"1","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.313789167Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"0","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.314546924Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.314578767Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.315192689Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"2","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.315775147Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"2","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[0]R flush_numbers[0..12]: [189, 196, 207, 207, 207, 204, 161, 204, 205, 197, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316321553Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[0]R generation[0..12]: [110, 113, 117, 117, 117, 116, 91, 116, 116, 114, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316341305Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316356216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[1]R flush_numbers[0..12]: [189, 196, 207, 207, 207, 204, 161, 204, 205, 197, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316369987Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[1]R generation[0..12]: [110, 113, 117, 117, 117, 116, 91, 116, 116, 114, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316383229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.31639717Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[2]R flush_numbers[0..12]: [189, 196, 207, 207, 207, 204, 161, 204, 205, 197, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316411002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[2]R generation[0..12]: [110, 113, 117, 117, 117, 116, 91, 116, 116, 114, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316424223Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316440054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:33:49.316455756Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316477508Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.3164931Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316508732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"":"downstairs","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316527723Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316542615Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"0","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316559037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"1","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316574898Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"client":"2","":"downstairs","session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.31659075Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316605541Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"c4117416-0398-4301-95f3-360c25e32b10 is now active with session: 03dc8ca8-c757-4063-9d77-7f3306c21078","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316621153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316635754Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662,"session_id":"03dc8ca8-c757-4063-9d77-7f3306c21078"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:49.316650386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 567 len 36864 data: 10 9 11 11 10 13 12 10 11 03/30 Read block 287 len 28672 04/30 Write block 501 len 12288 data: 7 7 7 05/30 Read block 465 len 12288 06/30 Read block 442 len 12288 07/30 Write block 455 len 20480 data: 8 9 10 10 8 08/30 Read block 358 len 36864 09/30 Write block 332 len 4096 data: 12 10/30 Flush 11/30 Write block 310 len 16384 data: 10 10 10 8 12/30 Write block 570 len 28672 data: 12 11 14 13 11 12 9 13/30 Write block 487 len 32768 data: 8 11 9 7 6 5 5 7 14/30 Read block 136 len 16384 15/30 Read block 138 len 4096 16/30 Write block 542 len 36864 data: 10 10 10 10 10 11 16 17 16 17/30 Write block 175 len 28672 data: 9 9 6 5 7 7 7 18/30 Write block 384 len 8192 data: 13 14 19/30 Read block 426 len 20480 20/30 Read block 449 len 8192 21/30 Flush 22/30 Write block 40 len 32768 data: 10 11 12 12 9 7 8 8 23/30 Read block 452 len 8192 24/30 Write block 147 len 12288 data: 9 7 12 25/30 Read block 514 len 32768 26/30 Read block 152 len 32768 27/30 Read block 564 len 40960 28/30 Write block 120 len 16384 data: 16 16 14 13 29/30 Write block 449 len 36864 data: 5 5 6 10 10 10 9 10 11 30/30 Read block 257 len 8192 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:49.337383347Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 117 117 117 207 207 207 T T T 3 060-079 117 117 117 207 207 207 F F F 4 080-099 117 117 117 207 207 207 F F F 5 100-119 116 116 116 204 204 204 F F F 6 120-139 91 91 91 161 161 161 T T T 7 140-159 116 116 116 204 204 204 T T T 8 160-179 118 118 118 210 210 210 F F F 9 180-199 118 118 118 210 210 210 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 117 117 117 207 207 207 F F F 12 240-259 116 116 116 202 202 202 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 116 116 116 206 206 206 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 117 117 117 207 207 207 F F F 19 380-399 118 118 118 210 210 210 F F F 20 400-419 115 115 115 201 201 201 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 118 118 118 209 209 209 T T T 23 460-479 112 112 112 192 192 192 F F F 24 480-499 118 118 118 210 210 210 F F F 25 500-519 118 118 118 209 209 209 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 118 118 118 210 210 210 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 115 115 115 201 201 201 F F F Max gen: 118, Max flush: 210 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 189 189 189 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 119 119 119 211 211 211 F F F 3 060-079 117 117 117 207 207 207 F F F 4 080-099 117 117 117 207 207 207 F F F 5 100-119 116 116 116 204 204 204 F F F 6 120-139 119 119 119 211 211 211 F F F 7 140-159 119 119 119 211 211 211 F F F 8 160-179 118 118 118 210 210 210 F F F 9 180-199 118 118 118 210 210 210 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 117 117 117 207 207 207 F F F 12 240-259 116 116 116 202 202 202 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 116 116 116 206 206 206 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 109 109 109 187 187 187 F F F 18 360-379 117 117 117 207 207 207 F F F 19 380-399 118 118 118 210 210 210 F F F 20 400-419 115 115 115 201 201 201 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 119 119 119 211 211 211 F F F 23 460-479 112 112 112 192 192 192 F F F 24 480-499 118 118 118 210 210 210 F F F 25 500-519 118 118 118 209 209 209 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 118 118 118 210 210 210 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 115 115 115 201 201 201 F F F Max gen: 119, Max flush: 211 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.576164308Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.620741604Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.621658856Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.621713002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.621732053Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.621750175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670} {"msg":"Crucible 4c15f8f2-c9aa-4c7c-83ec-47c78d2bd68d has session id: 8a8286ab-2c9c-4fae-857a-0c198a6a3ea9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622259457Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Upstairs opts: Upstairs UUID: 4c15f8f2-c9aa-4c7c-83ec-47c78d2bd68d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622298431Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Crucible stats registered with UUID: 4c15f8f2-c9aa-4c7c-83ec-47c78d2bd68d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622322963Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:53.622343675Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622768198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622798961Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670} {"msg":"4c15f8f2-c9aa-4c7c-83ec-47c78d2bd68d active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622855517Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622881709Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"0","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622907512Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"0","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622922313Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622936415Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622949386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"2","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622963568Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"2","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.622976729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.62299018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.623004462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"ds_connection connected from Ok(127.0.0.1:43016)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.623101082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64421)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.623143216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:53.623176619Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:53.623204202Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:33:53.623371759Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.623520024Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.623906453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"2","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.623946777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"0","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.624583051Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"0","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.624610764Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"2","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.62516618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:33:53.625249668Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.62417391Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59345)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.624339446Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.625573651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626116616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[0]R flush_numbers[0..12]: [189, 196, 211, 207, 207, 204, 211, 211, 210, 210, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626787824Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[0]R generation[0..12]: [110, 113, 119, 117, 117, 116, 119, 119, 118, 118, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626811806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626828618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[1]R flush_numbers[0..12]: [189, 196, 211, 207, 207, 204, 211, 211, 210, 210, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.62684671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[1]R generation[0..12]: [110, 113, 119, 117, 117, 116, 119, 119, 118, 118, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626860601Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626874302Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[2]R flush_numbers[0..12]: [189, 196, 211, 207, 207, 204, 211, 211, 210, 210, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626887734Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[2]R generation[0..12]: [110, 113, 119, 117, 117, 116, 119, 119, 118, 118, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626901235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626914926Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2025-02-27T17:34:03.626927448Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.62694658Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626959051Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626971432Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"":"downstairs","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626984083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.626995694Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"0","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.627008496Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"1","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.627021087Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"client":"2","":"downstairs","session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.627036449Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.62704852Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 4c15f8f2-c9aa-4c7c-83ec-47c78d2bd68d is now active with session: 8a8286ab-2c9c-4fae-857a-0c198a6a3ea9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.627063871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.627075753Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670,"session_id":"8a8286ab-2c9c-4fae-857a-0c198a6a3ea9"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:03.627087534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 12288 data: 9 10 11 02/30 Write block 476 len 12288 data: 7 5 4 03/30 Write block 501 len 8192 data: 8 8 04/30 Write block 70 len 20480 data: 9 10 8 9 10 05/30 Write block 10 len 24576 data: 11 9 11 13 13 15 06/30 Read block 371 len 8192 07/30 Write block 137 len 8192 data: 11 11 08/30 Read block 354 len 32768 09/30 Write block 186 len 8192 data: 9 6 10/30 Write block 501 len 40960 data: 9 9 8 6 7 7 5 5 8 8 11/30 Write block 415 len 32768 data: 11 9 11 13 11 10 10 11 12/30 Write block 184 len 4096 data: 9 13/30 Write block 429 len 20480 data: 9 10 11 8 6 14/30 Write block 62 len 8192 data: 9 9 15/30 Flush 16/30 Write block 97 len 28672 data: 3 4 4 4 5 5 6 17/30 Write block 76 len 4096 data: 13 18/30 Write block 132 len 32768 data: 8 10 12 11 12 12 12 13 19/30 Write block 491 len 20480 data: 7 6 6 8 7 20/30 Write block 583 len 4096 data: 7 21/30 Write block 49 len 24576 data: 6 6 8 9 7 11 22/30 Read block 468 len 40960 23/30 Read block 131 len 28672 24/30 Flush 25/30 Flush 26/30 Write block 354 len 28672 data: 7 7 7 9 7 5 7 27/30 Read block 491 len 40960 28/30 Write block 406 len 8192 data: 8 8 29/30 Write block 423 len 8192 data: 9 6 30/30 Read block 532 len 40960 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 4 3 5 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:03.643863176Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 212 212 212 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 120 120 120 213 213 213 F F F 3 060-079 120 120 120 213 213 213 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 120 120 120 213 213 213 F F F 6 120-139 120 120 120 213 213 213 F F F 7 140-159 119 119 119 211 211 211 F F F 8 160-179 118 118 118 210 210 210 F F F 9 180-199 120 120 120 212 212 212 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 117 117 117 207 207 207 F F F 12 240-259 116 116 116 202 202 202 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 116 116 116 206 206 206 F F F 15 300-319 120 120 120 212 212 212 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 109 109 109 187 187 187 T T T 18 360-379 117 117 117 207 207 207 T T T 19 380-399 118 118 118 210 210 210 F F F 20 400-419 120 120 120 212 212 212 T T T 21 420-439 120 120 120 212 212 212 T T T 22 440-459 119 119 119 211 211 211 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 120 120 120 213 213 213 F F F 25 500-519 120 120 120 212 212 212 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 118 118 118 210 210 210 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 120 120 120 213 213 213 F F F Max gen: 120, Max flush: 213 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 120 120 120 212 212 212 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 120 120 120 213 213 213 F F F 3 060-079 120 120 120 213 213 213 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 120 120 120 213 213 213 F F F 6 120-139 120 120 120 213 213 213 F F F 7 140-159 119 119 119 211 211 211 F F F 8 160-179 118 118 118 210 210 210 F F F 9 180-199 120 120 120 212 212 212 F F F 10 200-219 112 112 112 194 194 194 F F F 11 220-239 117 117 117 207 207 207 F F F 12 240-259 116 116 116 202 202 202 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 116 116 116 206 206 206 F F F 15 300-319 120 120 120 212 212 212 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 121 121 121 214 214 214 F F F 18 360-379 121 121 121 214 214 214 F F F 19 380-399 118 118 118 210 210 210 F F F 20 400-419 121 121 121 214 214 214 F F F 21 420-439 121 121 121 214 214 214 F F F 22 440-459 119 119 119 211 211 211 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 120 120 120 213 213 213 F F F 25 500-519 120 120 120 212 212 212 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 118 118 118 210 210 210 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 120 120 120 213 213 213 F F F Max gen: 121, Max flush: 214 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.881579029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.928046015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.928793671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.928836125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.928851497Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.928866098Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678} {"msg":"Crucible a4d3d9d5-3e57-4f9a-a9ab-1305cfa2491b has session id: 8fac0659-c152-47bc-8826-760d1705b1bb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.929433255Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Upstairs opts: Upstairs UUID: a4d3d9d5-3e57-4f9a-a9ab-1305cfa2491b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.92947831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Crucible stats registered with UUID: a4d3d9d5-3e57-4f9a-a9ab-1305cfa2491b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.929506403Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:07.929527735Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930088211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930125845Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678} {"msg":"a4d3d9d5-3e57-4f9a-a9ab-1305cfa2491b active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930185791Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930209743Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"0","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930250658Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"0","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930266639Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930281771Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930310244Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"2","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930338287Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"2","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930362919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930388161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930410964Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:46306)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930452328Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:55600)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930481531Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:07.930716174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:07.930795153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:07.930832306Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.930881121Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.93116518Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"2","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.931193203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"0","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.931704864Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"0","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.931808505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"2","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.932293404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:07.932382122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.931672421Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:32829)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.931808285Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.932964671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.933493535Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[0]R flush_numbers[0..12]: [212, 196, 213, 213, 213, 213, 213, 211, 210, 212, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934110237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[0]R generation[0..12]: [120, 113, 120, 120, 120, 120, 120, 119, 118, 120, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934135269Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934150861Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[1]R flush_numbers[0..12]: [212, 196, 213, 213, 213, 213, 213, 211, 210, 212, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934164992Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[1]R generation[0..12]: [120, 113, 120, 120, 120, 120, 120, 119, 118, 120, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934178754Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934192595Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[2]R flush_numbers[0..12]: [212, 196, 213, 213, 213, 213, 213, 211, 210, 212, 194, 207]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934210917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[2]R generation[0..12]: [120, 113, 120, 120, 120, 120, 120, 119, 118, 120, 112, 117]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934224869Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.93423878Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934251641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934264443Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934276774Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934289165Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"":"downstairs","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934301886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934313667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"0","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934332189Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"1","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934344851Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"client":"2","":"downstairs","session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934357532Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934369333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"a4d3d9d5-3e57-4f9a-a9ab-1305cfa2491b is now active with session: 8fac0659-c152-47bc-8826-760d1705b1bb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934381605Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934393316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678,"session_id":"8fac0659-c152-47bc-8826-760d1705b1bb"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:17.934405237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 8192 data: 9 12 02/30 Write block 58 len 16384 data: 12 10 11 11 03/30 Read block 287 len 40960 04/30 Write block 0 len 16384 data: 3 4 6 10 05/30 Write block 287 len 24576 data: 10 9 7 8 10 9 06/30 Flush 07/30 Write block 170 len 28672 data: 8 7 7 7 10 10 10 08/30 Read block 345 len 32768 09/30 Read block 76 len 20480 10/30 Write block 253 len 16384 data: 20 16 11 11 11/30 Write block 235 len 8192 data: 11 11 12/30 Read block 378 len 32768 13/30 Read block 393 len 16384 14/30 Read block 502 len 16384 15/30 Read block 355 len 24576 16/30 Read block 429 len 20480 17/30 Write block 579 len 4096 data: 8 18/30 Write block 75 len 8192 data: 11 14 19/30 Read block 484 len 32768 20/30 Read block 87 len 28672 21/30 Read block 351 len 32768 22/30 Write block 226 len 32768 data: 12 10 11 9 11 11 9 10 23/30 Read block 129 len 28672 24/30 Read block 212 len 12288 25/30 Write block 491 len 28672 data: 8 7 7 9 8 8 8 26/30 Write block 194 len 32768 data: 10 12 8 10 8 11 10 10 27/30 Write block 249 len 24576 data: 15 16 16 17 21 17 28/30 Read block 33 len 28672 29/30 Write block 582 len 12288 data: 7 8 9 30/30 Write block 381 len 32768 data: 8 10 12 14 15 12 11 11 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 7 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:17.953923996Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 215 215 215 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 122 122 122 215 215 215 F F F 3 060-079 122 122 122 215 215 215 T T T 4 080-099 120 120 120 213 213 213 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 120 120 120 213 213 213 F F F 7 140-159 119 119 119 211 211 211 F F F 8 160-179 118 118 118 210 210 210 T T T 9 180-199 120 120 120 212 212 212 T T T 10 200-219 112 112 112 194 194 194 T T T 11 220-239 117 117 117 207 207 207 T T T 12 240-259 116 116 116 202 202 202 T T T 13 260-279 113 113 113 196 196 196 F F F 14 280-299 122 122 122 215 215 215 F F F 15 300-319 120 120 120 212 212 212 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 121 121 121 214 214 214 F F F 18 360-379 121 121 121 214 214 214 F F F 19 380-399 118 118 118 210 210 210 T T T 20 400-419 121 121 121 214 214 214 F F F 21 420-439 121 121 121 214 214 214 F F F 22 440-459 119 119 119 211 211 211 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 120 120 120 213 213 213 T T T 25 500-519 120 120 120 212 212 212 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 118 118 118 210 210 210 F F F 28 560-579 118 118 118 210 210 210 T T T 29 580-599 120 120 120 213 213 213 T T T Max gen: 122, Max flush: 215 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 215 215 215 F F F 1 020-039 113 113 113 196 196 196 F F F 2 040-059 122 122 122 215 215 215 F F F 3 060-079 123 123 123 216 216 216 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 122 122 122 215 215 215 F F F 6 120-139 120 120 120 213 213 213 F F F 7 140-159 119 119 119 211 211 211 F F F 8 160-179 123 123 123 216 216 216 F F F 9 180-199 123 123 123 216 216 216 F F F 10 200-219 123 123 123 216 216 216 F F F 11 220-239 123 123 123 216 216 216 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 122 122 122 215 215 215 F F F 15 300-319 120 120 120 212 212 212 F F F 16 320-339 118 118 118 209 209 209 F F F 17 340-359 121 121 121 214 214 214 F F F 18 360-379 121 121 121 214 214 214 F F F 19 380-399 123 123 123 216 216 216 F F F 20 400-419 121 121 121 214 214 214 F F F 21 420-439 121 121 121 214 214 214 F F F 22 440-459 119 119 119 211 211 211 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 120 120 120 212 212 212 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 118 118 118 210 210 210 F F F 28 560-579 123 123 123 216 216 216 F F F 29 580-599 123 123 123 216 216 216 F F F Max gen: 123, Max flush: 216 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.226672112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.275362163Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.27622351Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.276282516Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.276301227Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.276319009Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686} {"msg":"Crucible 6230b5da-1146-421e-bca3-f05d75a18285 has session id: 13cd2bc8-a599-4655-bfae-d2a5ac6372b8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.27702478Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Upstairs opts: Upstairs UUID: 6230b5da-1146-421e-bca3-f05d75a18285, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277105639Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Crucible stats registered with UUID: 6230b5da-1146-421e-bca3-f05d75a18285","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277150113Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:22.277176716Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277609819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277636152Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686} {"msg":"6230b5da-1146-421e-bca3-f05d75a18285 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.27771642Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277734262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"0","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277758024Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"0","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277771786Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"1","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277786337Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"1","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.27781405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"2","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277843133Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"2","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277870176Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277897148Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277923011Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:51655)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277954284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:53316)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.277988348Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49588)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.278323422Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.278852575Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"0","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.278883988Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"2","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.2790023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"1","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.27960101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"1","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.279743815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"0","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.279773298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"2","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280188759Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280281059Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[0]R flush_numbers[0..12]: [215, 196, 215, 216, 213, 215, 213, 211, 216, 216, 216, 216]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280375989Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[0]R generation[0..12]: [122, 113, 122, 123, 120, 122, 120, 119, 123, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.28039463Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280409892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[1]R flush_numbers[0..12]: [215, 196, 215, 216, 213, 215, 213, 211, 216, 216, 216, 216]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280423743Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[1]R generation[0..12]: [122, 113, 122, 123, 120, 122, 120, 119, 123, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280437175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280450906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[2]R flush_numbers[0..12]: [215, 196, 215, 216, 213, 215, 213, 211, 216, 216, 216, 216]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280464427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[2]R generation[0..12]: [122, 113, 122, 123, 120, 122, 120, 119, 123, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280477739Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.28049138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:34:22.280504041Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280523863Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280536505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280552986Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"":"downstairs","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280566268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280577839Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"0","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.28059071Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"1","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280603491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"client":"2","":"downstairs","session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:34:22.280616603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280632424Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"6230b5da-1146-421e-bca3-f05d75a18285 is now active with session: 13cd2bc8-a599-4655-bfae-d2a5ac6372b8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280645225Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280656917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686,"session_id":"13cd2bc8-a599-4655-bfae-d2a5ac6372b8"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:22.280668618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 325 len 12288 data: 12 11 10 02/30 Read block 376 len 12288 03/30 Read block 474 len 36864 04/30 Read block 107 len 8192 05/30 Flush 06/30 Flush 07/30 Write block 296 len 32768 data: 6 8 8 10 13 13 16 15 08/30 Write block 409 len 36864 data: 13 13 9 13 13 12 12 10 12 09/30 Write block 41 len 24576 data: 12 13 13 10 8 9 10/30 Read block 119 len 28672 11/30 Read block 332 len 40960 12/30 Write block 108 len 28672 data: 10 13 13 11 11 9 7 13/30 Read block 345 len 16384 14/30 Read block 320 len 24576 15/30 Write block 207 len 40960 data: 7 7 9 10 11 12 13 11 13 13 16/30 Write block 307 len 24576 data: 9 10 11 12 11 11 17/30 Flush 18/30 Write block 27 len 4096 data: 9 19/30 Read block 197 len 4096 20/30 Flush 21/30 Write block 64 len 36864 data: 8 8 7 8 9 9 10 11 9 22/30 Read block 224 len 4096 23/30 Read block 419 len 40960 24/30 Flush 25/30 Write block 307 len 4096 data: 10 26/30 Write block 423 len 40960 data: 10 7 9 11 10 10 10 11 12 9 27/30 Write block 294 len 40960 data: 6 7 7 9 9 11 14 14 17 16 28/30 Write block 335 len 24576 data: 11 12 8 8 6 6 29/30 Read block 177 len 36864 30/30 Read block 417 len 20480 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:22.301038062Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 215 215 215 F F F 1 020-039 124 124 124 220 220 220 F F F 2 040-059 124 124 124 219 219 219 F F F 3 060-079 124 124 124 221 221 221 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 124 124 124 219 219 219 F F F 6 120-139 120 120 120 213 213 213 F F F 7 140-159 119 119 119 211 211 211 F F F 8 160-179 123 123 123 216 216 216 F F F 9 180-199 123 123 123 216 216 216 F F F 10 200-219 124 124 124 219 219 219 F F F 11 220-239 123 123 123 216 216 216 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 124 124 124 219 219 219 T T T 15 300-319 124 124 124 219 219 219 T T T 16 320-339 124 124 124 217 217 217 T T T 17 340-359 121 121 121 214 214 214 T T T 18 360-379 121 121 121 214 214 214 F F F 19 380-399 123 123 123 216 216 216 F F F 20 400-419 124 124 124 219 219 219 F F F 21 420-439 121 121 121 214 214 214 T T T 22 440-459 119 119 119 211 211 211 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 120 120 120 212 212 212 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 118 118 118 210 210 210 F F F 28 560-579 123 123 123 216 216 216 F F F 29 580-599 123 123 123 216 216 216 F F F Max gen: 124, Max flush: 221 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 215 215 215 F F F 1 020-039 124 124 124 220 220 220 F F F 2 040-059 124 124 124 219 219 219 F F F 3 060-079 124 124 124 221 221 221 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 124 124 124 219 219 219 F F F 6 120-139 120 120 120 213 213 213 F F F 7 140-159 119 119 119 211 211 211 F F F 8 160-179 123 123 123 216 216 216 F F F 9 180-199 123 123 123 216 216 216 F F F 10 200-219 124 124 124 219 219 219 F F F 11 220-239 123 123 123 216 216 216 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 113 113 113 196 196 196 F F F 14 280-299 125 125 125 222 222 222 F F F 15 300-319 125 125 125 222 222 222 F F F 16 320-339 125 125 125 222 222 222 F F F 17 340-359 125 125 125 222 222 222 F F F 18 360-379 121 121 121 214 214 214 F F F 19 380-399 123 123 123 216 216 216 F F F 20 400-419 124 124 124 219 219 219 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 119 119 119 211 211 211 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 120 120 120 212 212 212 F F F 26 520-539 112 112 112 192 192 192 F F F 27 540-559 118 118 118 210 210 210 F F F 28 560-579 123 123 123 216 216 216 F F F 29 580-599 123 123 123 216 216 216 F F F Max gen: 125, Max flush: 222 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.547829611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.592309627Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.59313025Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.593183015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.593201037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.593219779Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694} {"msg":"Crucible ea860d24-805f-4de9-9aa4-928939b386a5 has session id: 5f8d37ac-b340-43a2-a690-bb8a47283373","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.593756533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Upstairs opts: Upstairs UUID: ea860d24-805f-4de9-9aa4-928939b386a5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.593795647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Crucible stats registered with UUID: ea860d24-805f-4de9-9aa4-928939b386a5","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.593817849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:36.593838311Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594234001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594257174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694} {"msg":"ea860d24-805f-4de9-9aa4-928939b386a5 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594329341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594355454Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"0","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594381126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"0","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594404528Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"1","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594432221Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"1","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594457124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594484287Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594510619Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594537392Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594563575Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"ds_connection connected from Ok(127.0.0.1:60690)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594588917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"ds_connection connected from Ok(127.0.0.1:49857)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594613189Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:36.594642112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:36.594669435Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:36.594756444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.594789787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.595267415Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"0","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.595304849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"1","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.595952865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"0","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.595991849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"1","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.596643994Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:36.596679798Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.596009951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"ds_connection connected from Ok(127.0.0.1:40897)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.596345994Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.597208621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.597749256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[0]R flush_numbers[0..12]: [215, 220, 219, 221, 213, 219, 213, 211, 216, 216, 219, 216]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598359187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[0]R generation[0..12]: [122, 124, 124, 124, 120, 124, 120, 119, 123, 123, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.59838017Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598395041Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[1]R flush_numbers[0..12]: [215, 220, 219, 221, 213, 219, 213, 211, 216, 216, 219, 216]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598413873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[1]R generation[0..12]: [122, 124, 124, 124, 120, 124, 120, 119, 123, 123, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598427664Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598441296Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[2]R flush_numbers[0..12]: [215, 220, 219, 221, 213, 219, 213, 211, 216, 216, 219, 216]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598454697Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[2]R generation[0..12]: [122, 124, 124, 124, 120, 124, 120, 119, 123, 123, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598468078Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.59848159Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598494251Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598507292Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598519934Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598538555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"":"downstairs","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598551677Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598563258Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"0","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598576259Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"1","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.59858896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"client":"2","":"downstairs","session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598604142Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598616193Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"ea860d24-805f-4de9-9aa4-928939b386a5 is now active with session: 5f8d37ac-b340-43a2-a690-bb8a47283373","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598628524Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598640276Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694,"session_id":"5f8d37ac-b340-43a2-a690-bb8a47283373"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:34:46.598652397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 9 len 32768 data: 11 12 10 12 14 14 16 14 03/30 Read block 86 len 8192 04/30 Write block 120 len 8192 data: 17 17 05/30 Read block 335 len 16384 06/30 Flush 07/30 Write block 383 len 4096 data: 13 08/30 Read block 273 len 16384 09/30 Write block 590 len 28672 data: 10 11 9 10 8 5 4 10/30 Write block 273 len 12288 data: 7 7 6 11/30 Write block 223 len 24576 data: 15 15 15 13 11 12 12/30 Read block 128 len 20480 13/30 Read block 509 len 12288 14/30 Read block 496 len 40960 15/30 Read block 322 len 4096 16/30 Write block 58 len 4096 data: 13 17/30 Write block 290 len 16384 data: 9 11 10 7 18/30 Write block 179 len 12288 data: 8 8 8 19/30 Write block 195 len 8192 data: 13 9 20/30 Read block 454 len 8192 21/30 Read block 174 len 36864 22/30 Write block 321 len 28672 data: 18 16 16 15 13 12 11 23/30 Read block 80 len 32768 24/30 Read block 331 len 24576 25/30 Read block 74 len 32768 26/30 Write block 359 len 28672 data: 6 8 6 6 7 9 9 27/30 Read block 76 len 4096 28/30 Write block 537 len 36864 data: 4 8 7 7 9 11 11 11 11 29/30 Write block 117 len 32768 data: 12 14 13 18 18 15 14 12 30/30 Write block 86 len 16384 data: 14 14 14 15 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 1 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:34:46.617248793Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 224 224 224 F F F 1 020-039 124 124 124 220 220 220 F F F 2 040-059 124 124 124 219 219 219 T T T 3 060-079 124 124 124 221 221 221 F F F 4 080-099 120 120 120 213 213 213 T T T 5 100-119 124 124 124 219 219 219 T T T 6 120-139 126 126 126 224 224 224 T T T 7 140-159 119 119 119 211 211 211 F F F 8 160-179 123 123 123 216 216 216 T T T 9 180-199 123 123 123 216 216 216 T T T 10 200-219 124 124 124 219 219 219 F F F 11 220-239 123 123 123 216 216 216 T T T 12 240-259 123 123 123 216 216 216 F F F 13 260-279 113 113 113 196 196 196 T T T 14 280-299 125 125 125 222 222 222 T T T 15 300-319 125 125 125 222 222 222 F F F 16 320-339 125 125 125 222 222 222 T T T 17 340-359 125 125 125 222 222 222 T T T 18 360-379 121 121 121 214 214 214 T T T 19 380-399 123 123 123 216 216 216 T T T 20 400-419 124 124 124 219 219 219 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 119 119 119 211 211 211 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 120 120 120 212 212 212 F F F 26 520-539 112 112 112 192 192 192 T T T 27 540-559 118 118 118 210 210 210 T T T 28 560-579 123 123 123 216 216 216 F F F 29 580-599 123 123 123 216 216 216 T T T Max gen: 126, Max flush: 224 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 224 224 224 F F F 1 020-039 124 124 124 220 220 220 F F F 2 040-059 127 127 127 225 225 225 F F F 3 060-079 124 124 124 221 221 221 F F F 4 080-099 127 127 127 225 225 225 F F F 5 100-119 127 127 127 225 225 225 F F F 6 120-139 127 127 127 225 225 225 F F F 7 140-159 119 119 119 211 211 211 F F F 8 160-179 127 127 127 225 225 225 F F F 9 180-199 127 127 127 225 225 225 F F F 10 200-219 124 124 124 219 219 219 F F F 11 220-239 127 127 127 225 225 225 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 127 127 127 225 225 225 F F F 14 280-299 127 127 127 225 225 225 F F F 15 300-319 125 125 125 222 222 222 F F F 16 320-339 127 127 127 225 225 225 F F F 17 340-359 127 127 127 225 225 225 F F F 18 360-379 127 127 127 225 225 225 F F F 19 380-399 127 127 127 225 225 225 F F F 20 400-419 124 124 124 219 219 219 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 119 119 119 211 211 211 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 120 120 120 212 212 212 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 127 127 127 225 225 225 F F F 28 560-579 123 123 123 216 216 216 F F F 29 580-599 127 127 127 225 225 225 F F F Max gen: 127, Max flush: 225 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.91967646Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.964157126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.964944706Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.96498617Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.965003072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.965027354Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702} {"msg":"Crucible f1bbc59d-192d-4686-bc55-bc88ea380a9a has session id: cdf22805-8273-4c04-83db-55750e9a55c8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.965693452Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Upstairs opts: Upstairs UUID: f1bbc59d-192d-4686-bc55-bc88ea380a9a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.965727005Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Crucible stats registered with UUID: f1bbc59d-192d-4686-bc55-bc88ea380a9a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.965744967Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:00.965760498Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.96627369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966301633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702} {"msg":"f1bbc59d-192d-4686-bc55-bc88ea380a9a active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966361199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966391392Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"0","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966415584Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"0","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966429656Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"1","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966444157Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"1","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966457138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.96647129Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966484371Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966497942Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966511294Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:40375)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966591572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:00.966643807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:00.96667246Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61617)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966699113Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:00.966728126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.966883701Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.967297823Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"1","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.967339187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"0","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.967969001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"1","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.968024707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"0","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.968596124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:00.968681393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.968474492Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36233)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.968697264Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.969699906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.970279084Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[0]R flush_numbers[0..12]: [224, 220, 225, 221, 225, 225, 225, 211, 225, 225, 219, 225]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.970972674Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[0]R generation[0..12]: [126, 124, 127, 124, 127, 127, 127, 119, 127, 127, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.970997846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971014338Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[1]R flush_numbers[0..12]: [224, 220, 225, 221, 225, 225, 225, 211, 225, 225, 219, 225]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.97103275Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[1]R generation[0..12]: [126, 124, 127, 124, 127, 127, 127, 119, 127, 127, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971046831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971060803Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[2]R flush_numbers[0..12]: [224, 220, 225, 221, 225, 225, 225, 211, 225, 225, 219, 225]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971074534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[2]R generation[0..12]: [126, 124, 127, 124, 127, 127, 127, 119, 127, 127, 124, 127]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971087926Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971101537Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971114018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971126699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971140301Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971153112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"":"downstairs","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971166243Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971177804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"0","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971191126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"1","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971209298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"client":"2","":"downstairs","session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971222319Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.97123653Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"f1bbc59d-192d-4686-bc55-bc88ea380a9a is now active with session: cdf22805-8273-4c04-83db-55750e9a55c8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971249262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971260943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702,"session_id":"cdf22805-8273-4c04-83db-55750e9a55c8"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:10.971272794Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 320 len 32768 data: 15 19 17 17 16 14 13 12 02/30 Read block 398 len 20480 03/30 Write block 124 len 8192 data: 13 11 04/30 Read block 246 len 16384 05/30 Write block 236 len 12288 data: 12 10 11 06/30 Read block 212 len 20480 07/30 Read block 414 len 8192 08/30 Read block 495 len 12288 09/30 Flush 10/30 Flush 11/30 Write block 63 len 8192 data: 10 9 12/30 Write block 27 len 24576 data: 10 8 8 7 10 10 13/30 Read block 388 len 8192 14/30 Read block 513 len 36864 15/30 Write block 202 len 16384 data: 11 8 6 8 16/30 Write block 271 len 40960 data: 7 8 8 8 7 8 8 8 8 6 17/30 Read block 89 len 32768 18/30 Read block 389 len 24576 19/30 Write block 131 len 16384 data: 6 9 11 13 20/30 Read block 271 len 16384 21/30 Write block 153 len 28672 data: 11 10 9 7 8 8 6 22/30 Read block 171 len 28672 23/30 Write block 509 len 36864 data: 9 9 6 9 10 13 12 12 10 24/30 Write block 446 len 32768 data: 7 6 6 6 6 7 11 11 25/30 Read block 125 len 40960 26/30 Write block 95 len 16384 data: 4 4 4 5 27/30 Read block 430 len 12288 28/30 Read block 23 len 12288 29/30 Write block 446 len 4096 data: 8 30/30 Read block 399 len 28672 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 2 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 20 19 18 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:10.99234868Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 224 224 224 F F F 1 020-039 124 124 124 220 220 220 T T T 2 040-059 127 127 127 225 225 225 F F F 3 060-079 124 124 124 221 221 221 T T T 4 080-099 127 127 127 225 225 225 T T T 5 100-119 127 127 127 225 225 225 F F F 6 120-139 128 128 128 226 226 226 T T T 7 140-159 119 119 119 211 211 211 T T T 8 160-179 127 127 127 225 225 225 F F F 9 180-199 127 127 127 225 225 225 F F F 10 200-219 124 124 124 219 219 219 T T T 11 220-239 128 128 128 226 226 226 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 127 127 127 225 225 225 T T T 14 280-299 127 127 127 225 225 225 T T T 15 300-319 125 125 125 222 222 222 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 127 127 127 225 225 225 F F F 18 360-379 127 127 127 225 225 225 F F F 19 380-399 127 127 127 225 225 225 F F F 20 400-419 124 124 124 219 219 219 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 119 119 119 211 211 211 T T T 23 460-479 120 120 120 212 212 212 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 120 120 120 212 212 212 T T T 26 520-539 127 127 127 225 225 225 F F F 27 540-559 127 127 127 225 225 225 F F F 28 560-579 123 123 123 216 216 216 F F F 29 580-599 127 127 127 225 225 225 F F F Max gen: 128, Max flush: 226 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 224 224 224 F F F 1 020-039 129 129 129 227 227 227 F F F 2 040-059 127 127 127 225 225 225 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 129 129 129 227 227 227 F F F 5 100-119 127 127 127 225 225 225 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 129 129 129 227 227 227 F F F 8 160-179 127 127 127 225 225 225 F F F 9 180-199 127 127 127 225 225 225 F F F 10 200-219 129 129 129 227 227 227 F F F 11 220-239 128 128 128 226 226 226 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 129 129 129 227 227 227 F F F 14 280-299 129 129 129 227 227 227 F F F 15 300-319 125 125 125 222 222 222 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 127 127 127 225 225 225 F F F 18 360-379 127 127 127 225 225 225 F F F 19 380-399 127 127 127 225 225 225 F F F 20 400-419 124 124 124 219 219 219 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 129 129 129 227 227 227 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 129 129 129 227 227 227 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 127 127 127 225 225 225 F F F 28 560-579 123 123 123 216 216 216 F F F 29 580-599 127 127 127 225 225 225 F F F Max gen: 129, Max flush: 227 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.265305786Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.309887092Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.310575421Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.310624216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.310639108Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.310653409Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710} {"msg":"Crucible dd5328b2-14d3-4b68-9f27-ece652a22cfc has session id: 769ec6ef-0d92-4aa6-9f07-38b1531a74b9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311275082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Upstairs opts: Upstairs UUID: dd5328b2-14d3-4b68-9f27-ece652a22cfc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311313496Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Crucible stats registered with UUID: dd5328b2-14d3-4b68-9f27-ece652a22cfc","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311331758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:25.311344639Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311724247Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.31174816Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710} {"msg":"dd5328b2-14d3-4b68-9f27-ece652a22cfc active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311811186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311836589Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"0","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311866012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"0","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311881593Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"1","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311896695Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"1","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311918787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.31194558Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311970032Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.311998735Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.312023997Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35934)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.31204906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55983)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.31214627Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:25.312185404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:25.312214707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:25.312313217Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.312490315Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.312908557Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"1","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.3129357Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"0","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.313572544Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"1","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.313602267Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"0","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.314188006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:25.314275475Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.314439771Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41439)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.31462435Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.315582586Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316096468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[0]R flush_numbers[0..12]: [224, 227, 225, 227, 227, 225, 227, 227, 225, 225, 227, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316697229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[0]R generation[0..12]: [126, 129, 127, 129, 129, 127, 129, 129, 127, 127, 129, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316726082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316744903Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[1]R flush_numbers[0..12]: [224, 227, 225, 227, 227, 225, 227, 227, 225, 225, 227, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316761965Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[1]R generation[0..12]: [126, 129, 127, 129, 129, 127, 129, 129, 127, 127, 129, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316785398Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.3168037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[2]R flush_numbers[0..12]: [224, 227, 225, 227, 227, 225, 227, 227, 225, 225, 227, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316820491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[2]R generation[0..12]: [126, 129, 127, 129, 129, 127, 129, 129, 127, 127, 129, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316837343Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316854085Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316875747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316891998Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.31690758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316923381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"":"downstairs","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316939453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316953905Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"0","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316969976Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"1","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.316985808Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"client":"2","":"downstairs","session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.317001629Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.317016601Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"dd5328b2-14d3-4b68-9f27-ece652a22cfc is now active with session: 769ec6ef-0d92-4aa6-9f07-38b1531a74b9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.317033963Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.317048384Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710,"session_id":"769ec6ef-0d92-4aa6-9f07-38b1531a74b9"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:35.317063206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 156 len 32768 data: 8 9 9 7 6 7 7 8 02/30 Flush 03/30 Write block 347 len 28672 data: 5 7 7 8 6 7 7 04/30 Write block 360 len 8192 data: 9 7 05/30 Write block 116 len 4096 data: 10 06/30 Read block 102 len 36864 07/30 Read block 407 len 32768 08/30 Read block 442 len 12288 09/30 Flush 10/30 Write block 215 len 8192 data: 14 14 11/30 Write block 485 len 36864 data: 7 7 9 12 10 8 9 8 8 12/30 Read block 147 len 32768 13/30 Flush 14/30 Flush 15/30 Write block 350 len 32768 data: 9 7 8 8 8 8 8 10 16/30 Write block 368 len 20480 data: 10 12 10 13 12 17/30 Flush 18/30 Write block 550 len 16384 data: 17 13 11 12 19/30 Flush 20/30 Write block 445 len 24576 data: 5 9 7 7 7 7 21/30 Read block 258 len 36864 22/30 Read block 117 len 32768 23/30 Flush 24/30 Write block 99 len 12288 data: 5 5 6 25/30 Write block 216 len 8192 data: 15 14 26/30 Write block 352 len 4096 data: 9 27/30 Read block 90 len 32768 28/30 Read block 416 len 8192 29/30 Write block 505 len 28672 data: 8 8 6 6 10 10 7 30/30 Write block 22 len 16384 data: 8 7 6 7 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:35.335938029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 224 224 224 F F F 1 020-039 129 129 129 227 227 227 T T T 2 040-059 127 127 127 225 225 225 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 129 129 129 227 227 227 T T T 5 100-119 130 130 130 229 229 229 T T T 6 120-139 129 129 129 227 227 227 F F F 7 140-159 130 130 130 228 228 228 F F F 8 160-179 130 130 130 228 228 228 F F F 9 180-199 127 127 127 225 225 225 F F F 10 200-219 130 130 130 230 230 230 T T T 11 220-239 128 128 128 226 226 226 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 129 129 129 227 227 227 F F F 14 280-299 129 129 129 227 227 227 F F F 15 300-319 125 125 125 222 222 222 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 130 130 130 232 232 232 T T T 18 360-379 130 130 130 232 232 232 F F F 19 380-399 127 127 127 225 225 225 F F F 20 400-419 124 124 124 219 219 219 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 130 130 130 234 234 234 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 129 129 129 227 227 227 T T T 26 520-539 127 127 127 225 225 225 F F F 27 540-559 130 130 130 233 233 233 F F F 28 560-579 123 123 123 216 216 216 F F F 29 580-599 127 127 127 225 225 225 F F F Max gen: 130, Max flush: 234 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 224 224 224 F F F 1 020-039 131 131 131 235 235 235 F F F 2 040-059 127 127 127 225 225 225 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 131 131 131 235 235 235 F F F 5 100-119 131 131 131 235 235 235 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 130 130 130 228 228 228 F F F 8 160-179 130 130 130 228 228 228 F F F 9 180-199 127 127 127 225 225 225 F F F 10 200-219 131 131 131 235 235 235 F F F 11 220-239 128 128 128 226 226 226 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 129 129 129 227 227 227 F F F 14 280-299 129 129 129 227 227 227 F F F 15 300-319 125 125 125 222 222 222 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 131 131 131 235 235 235 F F F 18 360-379 130 130 130 232 232 232 F F F 19 380-399 127 127 127 225 225 225 F F F 20 400-419 124 124 124 219 219 219 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 130 130 130 234 234 234 F F F 23 460-479 120 120 120 212 212 212 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 130 130 130 233 233 233 F F F 28 560-579 123 123 123 216 216 216 F F F 29 580-599 127 127 127 225 225 225 F F F Max gen: 131, Max flush: 235 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.588409841Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.632654083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.633575256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.633619651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.633640943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.633656274Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718} {"msg":"Crucible 91367743-8cca-40cf-acbc-35a0bc0d93be has session id: 8c83e5c5-a616-4da9-abdd-866956a4efc9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.634347074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Upstairs opts: Upstairs UUID: 91367743-8cca-40cf-acbc-35a0bc0d93be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.634386648Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Crucible stats registered with UUID: 91367743-8cca-40cf-acbc-35a0bc0d93be","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.63440269Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:39.634415681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.634911141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.634942684Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718} {"msg":"91367743-8cca-40cf-acbc-35a0bc0d93be active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.63500489Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635021662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"0","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635042834Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"0","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635062006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635087919Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635112971Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"2","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635171187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"2","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.63519914Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635222182Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635246175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:39532)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635268337Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:39.635309931Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:39.635341334Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64219)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635385379Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:39.635508901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.635662647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.636119443Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"2","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.636147265Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"0","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.636761887Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"0","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.636912633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"2","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.637353727Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:39.637472249Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.636825214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37262)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.636957077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.638113524Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.638711594Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[0]R flush_numbers[0..12]: [224, 235, 225, 227, 235, 235, 227, 228, 228, 225, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639355979Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[0]R generation[0..12]: [126, 131, 127, 129, 131, 131, 129, 130, 130, 127, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639377001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639392263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[1]R flush_numbers[0..12]: [224, 235, 225, 227, 235, 235, 227, 228, 228, 225, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639406274Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[1]R generation[0..12]: [126, 131, 127, 129, 131, 131, 129, 130, 130, 127, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639420376Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639434347Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[2]R flush_numbers[0..12]: [224, 235, 225, 227, 235, 235, 227, 228, 228, 225, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639447908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[2]R generation[0..12]: [126, 131, 127, 129, 131, 131, 129, 130, 130, 127, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.63946125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639478982Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639492123Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:35:49.639505124Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639521386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639533797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"":"downstairs","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639546778Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.63955852Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"0","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639571411Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"1","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639584082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"client":"2","":"downstairs","session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639596914Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639608845Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"91367743-8cca-40cf-acbc-35a0bc0d93be is now active with session: 8c83e5c5-a616-4da9-abdd-866956a4efc9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639621186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639632807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718,"session_id":"8c83e5c5-a616-4da9-abdd-866956a4efc9"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:35:49.639644628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 11 len 32768 data: 11 13 15 15 17 15 12 10 03/30 Read block 255 len 36864 04/30 Write block 541 len 24576 data: 10 12 12 12 12 11 05/30 Read block 427 len 40960 06/30 Read block 506 len 16384 07/30 Read block 224 len 16384 08/30 Write block 22 len 32768 data: 9 8 7 8 7 11 9 9 09/30 Write block 306 len 16384 data: 13 11 11 12 10/30 Read block 446 len 8192 11/30 Flush 12/30 Write block 293 len 16384 data: 8 7 8 8 13/30 Write block 341 len 12288 data: 6 7 8 14/30 Write block 263 len 24576 data: 7 8 8 8 11 11 15/30 Read block 241 len 8192 16/30 Read block 411 len 4096 17/30 Read block 550 len 4096 18/30 Flush 19/30 Flush 20/30 Write block 49 len 12288 data: 7 7 9 21/30 Read block 26 len 36864 22/30 Flush 23/30 Write block 285 len 12288 data: 4 7 11 24/30 Read block 52 len 20480 25/30 Write block 402 len 20480 data: 8 7 9 8 9 26/30 Read block 448 len 12288 27/30 Write block 476 len 4096 data: 8 28/30 Write block 285 len 40960 data: 5 8 12 10 8 10 12 11 9 8 29/30 Write block 558 len 36864 data: 12 12 9 8 6 6 8 11 10 30/30 Write block 315 len 8192 data: 10 10 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 3 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:35:49.658610821Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 237 237 237 F F F 1 020-039 132 132 132 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 131 131 131 235 235 235 F F F 5 100-119 131 131 131 235 235 235 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 130 130 130 228 228 228 F F F 8 160-179 130 130 130 228 228 228 F F F 9 180-199 127 127 127 225 225 225 F F F 10 200-219 131 131 131 235 235 235 F F F 11 220-239 128 128 128 226 226 226 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 132 132 132 238 238 238 F F F 14 280-299 132 132 132 238 238 238 T T T 15 300-319 132 132 132 237 237 237 T T T 16 320-339 128 128 128 226 226 226 F F F 17 340-359 132 132 132 238 238 238 F F F 18 360-379 130 130 130 232 232 232 F F F 19 380-399 127 127 127 225 225 225 F F F 20 400-419 124 124 124 219 219 219 T T T 21 420-439 125 125 125 222 222 222 F F F 22 440-459 130 130 130 234 234 234 F F F 23 460-479 120 120 120 212 212 212 T T T 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 132 132 132 237 237 237 T T T 28 560-579 123 123 123 216 216 216 T T T 29 580-599 127 127 127 225 225 225 F F F Max gen: 132, Max flush: 240 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 237 237 237 F F F 1 020-039 132 132 132 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 131 131 131 235 235 235 F F F 5 100-119 131 131 131 235 235 235 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 130 130 130 228 228 228 F F F 8 160-179 130 130 130 228 228 228 F F F 9 180-199 127 127 127 225 225 225 F F F 10 200-219 131 131 131 235 235 235 F F F 11 220-239 128 128 128 226 226 226 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 132 132 132 238 238 238 F F F 14 280-299 133 133 133 241 241 241 F F F 15 300-319 133 133 133 241 241 241 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 132 132 132 238 238 238 F F F 18 360-379 130 130 130 232 232 232 F F F 19 380-399 127 127 127 225 225 225 F F F 20 400-419 133 133 133 241 241 241 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 130 130 130 234 234 234 F F F 23 460-479 133 133 133 241 241 241 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 133 133 133 241 241 241 F F F 28 560-579 133 133 133 241 241 241 F F F 29 580-599 127 127 127 225 225 225 F F F Max gen: 133, Max flush: 241 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.905053573Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.949328529Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.950253412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.950296127Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.950311148Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.950325089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726} {"msg":"Crucible 76ceee4e-0230-4422-8e37-bbec609b90f9 has session id: 57bb071f-eeee-4146-ae4f-b412313398b4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.950978445Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Upstairs opts: Upstairs UUID: 76ceee4e-0230-4422-8e37-bbec609b90f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.95102591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Crucible stats registered with UUID: 76ceee4e-0230-4422-8e37-bbec609b90f9","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951054433Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:03.951077235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951560974Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951590197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726} {"msg":"76ceee4e-0230-4422-8e37-bbec609b90f9 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951657984Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951684507Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"0","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951709729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"0","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951725091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"1","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951747363Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"1","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951772495Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951794478Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951807909Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951839822Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.951863194Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:03.951881136Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:03.951904729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:03.951932781Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33311)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.952022951Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.952061394Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:44309)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.952094228Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.952747534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"0","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.952783027Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"1","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.953437563Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"0","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.953474947Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"1","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.954075918Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:03.954113752Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.95399833Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:60679)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.954235054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.955261307Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.955812413Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[0]R flush_numbers[0..12]: [237, 237, 240, 227, 235, 235, 227, 228, 228, 225, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956430455Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[0]R generation[0..12]: [132, 132, 132, 129, 131, 131, 129, 130, 130, 127, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956460848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.95647712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[1]R flush_numbers[0..12]: [237, 237, 240, 227, 235, 235, 227, 228, 228, 225, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956491681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[1]R generation[0..12]: [132, 132, 132, 129, 131, 131, 129, 130, 130, 127, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956505773Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956520244Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[2]R flush_numbers[0..12]: [237, 237, 240, 227, 235, 235, 227, 228, 228, 225, 235, 226]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956534155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[2]R generation[0..12]: [132, 132, 132, 129, 131, 131, 129, 130, 130, 127, 131, 128]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956548097Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956561618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:36:13.95657441Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956589581Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956603453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956619884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"":"downstairs","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956633176Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956655358Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"0","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956668949Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"1","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.95668185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"client":"2","":"downstairs","session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956694762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956706753Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"76ceee4e-0230-4422-8e37-bbec609b90f9 is now active with session: 57bb071f-eeee-4146-ae4f-b412313398b4","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956719174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956730805Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726,"session_id":"57bb071f-eeee-4146-ae4f-b412313398b4"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:13.956742707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 36864 data: 13 13 12 12 17 18 18 14 12 02/30 Read block 176 len 28672 03/30 Write block 298 len 16384 data: 10 12 15 15 04/30 Write block 225 len 8192 data: 16 14 05/30 Write block 385 len 16384 data: 16 13 12 12 06/30 Write block 192 len 12288 data: 7 10 11 07/30 Read block 385 len 4096 08/30 Read block 48 len 36864 09/30 Write block 274 len 8192 data: 9 8 10/30 Read block 77 len 32768 11/30 Read block 534 len 12288 12/30 Flush 13/30 Write block 218 len 16384 data: 13 13 12 12 14/30 Read block 278 len 36864 15/30 Read block 218 len 4096 16/30 Read block 524 len 20480 17/30 Read block 430 len 32768 18/30 Read block 8 len 28672 19/30 Read block 83 len 40960 20/30 Flush 21/30 Write block 232 len 40960 data: 10 11 9 12 13 11 12 11 7 7 22/30 Read block 297 len 4096 23/30 Read block 87 len 8192 24/30 Write block 100 len 16384 data: 6 7 6 7 25/30 Write block 162 len 36864 data: 8 9 8 8 7 8 8 8 9 26/30 Write block 236 len 28672 data: 14 12 13 12 8 8 6 27/30 Write block 581 len 8192 data: 7 8 28/30 Read block 0 len 12288 29/30 Write block 159 len 40960 data: 8 7 8 9 10 9 9 8 9 9 30/30 Write block 354 len 36864 data: 9 9 9 11 8 7 10 8 7 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:13.977173987Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 237 237 237 F F F 1 020-039 132 132 132 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 131 131 131 235 235 235 F F F 5 100-119 131 131 131 235 235 235 T T T 6 120-139 129 129 129 227 227 227 F F F 7 140-159 130 130 130 228 228 228 T T T 8 160-179 130 130 130 228 228 228 T T T 9 180-199 134 134 134 242 242 242 F F F 10 200-219 134 134 134 243 243 243 F F F 11 220-239 134 134 134 243 243 243 T T T 12 240-259 123 123 123 216 216 216 T T T 13 260-279 134 134 134 242 242 242 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 132 132 132 238 238 238 T T T 18 360-379 130 130 130 232 232 232 T T T 19 380-399 134 134 134 242 242 242 F F F 20 400-419 133 133 133 241 241 241 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 130 130 130 234 234 234 F F F 23 460-479 133 133 133 241 241 241 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 134 134 134 242 242 242 F F F 28 560-579 133 133 133 241 241 241 F F F 29 580-599 127 127 127 225 225 225 T T T Max gen: 134, Max flush: 243 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 237 237 237 F F F 1 020-039 132 132 132 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 131 131 131 235 235 235 F F F 5 100-119 135 135 135 244 244 244 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 135 135 135 244 244 244 F F F 8 160-179 135 135 135 244 244 244 F F F 9 180-199 134 134 134 242 242 242 F F F 10 200-219 134 134 134 243 243 243 F F F 11 220-239 135 135 135 244 244 244 F F F 12 240-259 135 135 135 244 244 244 F F F 13 260-279 134 134 134 242 242 242 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 135 135 135 244 244 244 F F F 18 360-379 135 135 135 244 244 244 F F F 19 380-399 134 134 134 242 242 242 F F F 20 400-419 133 133 133 241 241 241 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 130 130 130 234 234 234 F F F 23 460-479 133 133 133 241 241 241 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 134 134 134 242 242 242 F F F 28 560-579 133 133 133 241 241 241 F F F 29 580-599 135 135 135 244 244 244 F F F Max gen: 135, Max flush: 244 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.236133764Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.28042197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.281160075Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.28121002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.281227381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.281246073Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734} {"msg":"Crucible 9ddc5330-6705-4187-bb3e-0ed45f328f77 has session id: cd2ab97b-08dc-4e16-aa70-533374044cbd","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.281894319Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Upstairs opts: Upstairs UUID: 9ddc5330-6705-4187-bb3e-0ed45f328f77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.281942453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Crucible stats registered with UUID: 9ddc5330-6705-4187-bb3e-0ed45f328f77","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.281971316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:28.282016721Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282451425Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282480438Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734} {"msg":"9ddc5330-6705-4187-bb3e-0ed45f328f77 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282534933Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282557706Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"0","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282579628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"0","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282593599Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"1","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282607961Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"1","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282620832Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282634353Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282647005Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282669057Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282688839Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:28.282823713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:28.282846845Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33151)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.282860986Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:28.282881198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.283049095Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:63302)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.28309953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.283568998Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"1","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.283746846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"0","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.284200231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"1","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.284305112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"0","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.284754537Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:28.284864798Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.283912082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61394)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.283998441Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.285191551Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.285758828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[0]R flush_numbers[0..12]: [237, 237, 240, 227, 235, 244, 227, 244, 244, 242, 243, 244]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286376201Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[0]R generation[0..12]: [132, 132, 132, 129, 131, 135, 129, 135, 135, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286398283Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286418685Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[1]R flush_numbers[0..12]: [237, 237, 240, 227, 235, 244, 227, 244, 244, 242, 243, 244]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286432906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[1]R generation[0..12]: [132, 132, 132, 129, 131, 135, 129, 135, 135, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286454678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286477331Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[2]R flush_numbers[0..12]: [237, 237, 240, 227, 235, 244, 227, 244, 244, 242, 243, 244]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286500713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[2]R generation[0..12]: [132, 132, 132, 129, 131, 135, 129, 135, 135, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286522785Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286540177Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:36:38.286563289Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286587812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286611845Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286626886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"":"downstairs","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286639827Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286673091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"0","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286696953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"1","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286724516Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"client":"2","":"downstairs","session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286748428Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.28676228Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"9ddc5330-6705-4187-bb3e-0ed45f328f77 is now active with session: cd2ab97b-08dc-4e16-aa70-533374044cbd","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286778181Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286799663Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734,"session_id":"cd2ab97b-08dc-4e16-aa70-533374044cbd"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:38.286822636Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 95 len 28672 data: 5 5 5 6 6 7 8 02/30 Read block 369 len 16384 03/30 Read block 416 len 28672 04/30 Read block 493 len 24576 05/30 Read block 294 len 24576 06/30 Read block 258 len 36864 07/30 Read block 436 len 24576 08/30 Read block 576 len 12288 09/30 Read block 336 len 24576 10/30 Read block 248 len 28672 11/30 Read block 56 len 36864 12/30 Read block 88 len 20480 13/30 Read block 172 len 12288 14/30 Write block 355 len 16384 data: 10 10 12 9 15/30 Write block 352 len 24576 data: 10 9 10 11 11 13 16/30 Write block 229 len 36864 data: 10 12 12 11 12 10 13 15 13 17/30 Read block 323 len 24576 18/30 Write block 445 len 24576 data: 6 10 8 8 8 8 19/30 Read block 415 len 16384 20/30 Read block 176 len 20480 21/30 Flush 22/30 Flush 23/30 Write block 340 len 4096 data: 7 24/30 Read block 77 len 16384 25/30 Flush 26/30 Write block 234 len 32768 data: 11 14 16 14 14 13 9 9 27/30 Read block 34 len 40960 28/30 Write block 233 len 32768 data: 13 12 15 17 15 15 14 10 29/30 Write block 258 len 12288 data: 10 12 11 30/30 Write block 577 len 8192 data: 9 9 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:38.307719043Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 237 237 237 F F F 1 020-039 132 132 132 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 136 136 136 245 245 245 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 135 135 135 244 244 244 F F F 8 160-179 135 135 135 244 244 244 F F F 9 180-199 134 134 134 242 242 242 F F F 10 200-219 134 134 134 243 243 243 F F F 11 220-239 136 136 136 245 245 245 T T T 12 240-259 135 135 135 244 244 244 T T T 13 260-279 134 134 134 242 242 242 T T T 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 135 135 135 244 244 244 F F F 19 380-399 134 134 134 242 242 242 F F F 20 400-419 133 133 133 241 241 241 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 136 136 136 245 245 245 F F F 23 460-479 133 133 133 241 241 241 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 134 134 134 242 242 242 F F F 28 560-579 133 133 133 241 241 241 T T T 29 580-599 135 135 135 244 244 244 F F F Max gen: 136, Max flush: 247 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 237 237 237 F F F 1 020-039 132 132 132 237 237 237 F F F 2 040-059 132 132 132 240 240 240 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 136 136 136 245 245 245 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 135 135 135 244 244 244 F F F 8 160-179 135 135 135 244 244 244 F F F 9 180-199 134 134 134 242 242 242 F F F 10 200-219 134 134 134 243 243 243 F F F 11 220-239 137 137 137 248 248 248 F F F 12 240-259 137 137 137 248 248 248 F F F 13 260-279 137 137 137 248 248 248 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 135 135 135 244 244 244 F F F 19 380-399 134 134 134 242 242 242 F F F 20 400-419 133 133 133 241 241 241 F F F 21 420-439 125 125 125 222 222 222 F F F 22 440-459 136 136 136 245 245 245 F F F 23 460-479 133 133 133 241 241 241 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 134 134 134 242 242 242 F F F 28 560-579 137 137 137 248 248 248 F F F 29 580-599 135 135 135 244 244 244 F F F Max gen: 137, Max flush: 248 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.546478612Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:42.590113763Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.590621834Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.622438683Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.623180288Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.623215161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.623230373Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.623244364Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742} {"msg":"Crucible 38b2ce67-83b4-4042-b367-0b5c003b15fb has session id: f8d7db72-7624-45f3-b1e3-fcc83fbc85f0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.623920072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Upstairs opts: Upstairs UUID: 38b2ce67-83b4-4042-b367-0b5c003b15fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.623948295Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Crucible stats registered with UUID: 38b2ce67-83b4-4042-b367-0b5c003b15fb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.623963086Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:42.623975478Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624359366Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624379679Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742} {"msg":"38b2ce67-83b4-4042-b367-0b5c003b15fb active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624448906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624468328Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"0","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624504141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"0","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624536564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"1","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624565627Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"1","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.62459129Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"2","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624618663Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"2","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624644915Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624671178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624697981Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59311)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624724553Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:53250)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624751866Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34348)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.624961167Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.625470189Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"1","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.625510523Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"2","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.625561028Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"0","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626129695Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"2","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626160628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"0","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.62617644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"1","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626747667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626781321Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[0]R flush_numbers[0..12]: [237, 237, 240, 227, 245, 245, 227, 244, 244, 242, 243, 248]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626805983Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[0]R generation[0..12]: [132, 132, 132, 129, 136, 136, 129, 135, 135, 134, 134, 137]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626820835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626835136Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[1]R flush_numbers[0..12]: [237, 237, 240, 227, 245, 245, 227, 244, 244, 242, 243, 248]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626848408Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[1]R generation[0..12]: [132, 132, 132, 129, 136, 136, 129, 135, 135, 134, 134, 137]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626861949Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626884651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[2]R flush_numbers[0..12]: [237, 237, 240, 227, 245, 245, 227, 244, 244, 242, 243, 248]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626910214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[2]R generation[0..12]: [132, 132, 132, 129, 136, 136, 129, 135, 135, 134, 134, 137]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626935256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.626955678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.62696893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:36:42.626994972Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627033316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627058579Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"":"downstairs","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.62707286Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627084681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"0","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627103143Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"1","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627126285Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"client":"2","":"downstairs","session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627150768Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.62717171Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"38b2ce67-83b4-4042-b367-0b5c003b15fb is now active with session: f8d7db72-7624-45f3-b1e3-fcc83fbc85f0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627185191Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627205533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742,"session_id":"f8d7db72-7624-45f3-b1e3-fcc83fbc85f0"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:42.627227406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 20480 data: 9 8 9 10 11 02/30 Write block 464 len 28672 data: 12 13 14 14 14 13 13 03/30 Write block 188 len 36864 data: 5 4 3 6 8 11 12 14 10 04/30 Read block 414 len 8192 05/30 Read block 136 len 4096 06/30 Flush 07/30 Write block 548 len 20480 data: 18 19 19 15 13 08/30 Read block 144 len 28672 09/30 Flush 10/30 Write block 53 len 20480 data: 8 12 13 12 13 11/30 Read block 385 len 4096 12/30 Write block 418 len 8192 data: 14 12 13/30 Read block 518 len 32768 14/30 Read block 565 len 16384 15/30 Write block 5 len 36864 data: 11 12 12 14 12 13 12 14 16 16/30 Read block 109 len 4096 17/30 Read block 455 len 20480 18/30 Read block 404 len 36864 19/30 Flush 20/30 Write block 276 len 16384 data: 9 9 9 9 21/30 Write block 26 len 32768 data: 8 12 10 10 8 11 11 11 22/30 Read block 315 len 40960 23/30 Read block 356 len 8192 24/30 Write block 418 len 40960 data: 15 13 11 11 12 11 8 10 12 11 25/30 Write block 226 len 24576 data: 15 12 13 11 13 13 26/30 Write block 360 len 36864 data: 11 9 8 8 10 10 9 10 11 27/30 Read block 155 len 8192 28/30 Read block 148 len 4096 29/30 Write block 215 len 12288 data: 15 16 15 30/30 Read block 538 len 32768 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 9 11 11 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:42.647404121Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 251 251 251 F F F 1 020-039 132 132 132 237 237 237 T T T 2 040-059 138 138 138 251 251 251 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 136 136 136 245 245 245 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 138 138 138 249 249 249 F F F 8 160-179 138 138 138 249 249 249 F F F 9 180-199 138 138 138 249 249 249 F F F 10 200-219 134 134 134 243 243 243 T T T 11 220-239 137 137 137 248 248 248 T T T 12 240-259 137 137 137 248 248 248 F F F 13 260-279 137 137 137 248 248 248 T T T 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 135 135 135 244 244 244 T T T 19 380-399 134 134 134 242 242 242 F F F 20 400-419 138 138 138 251 251 251 T T T 21 420-439 125 125 125 222 222 222 T T T 22 440-459 136 136 136 245 245 245 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 138 138 138 250 250 250 F F F 28 560-579 137 137 137 248 248 248 F F F 29 580-599 135 135 135 244 244 244 F F F Max gen: 138, Max flush: 251 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 251 251 251 F F F 1 020-039 139 139 139 252 252 252 F F F 2 040-059 138 138 138 251 251 251 F F F 3 060-079 129 129 129 227 227 227 F F F 4 080-099 136 136 136 245 245 245 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 138 138 138 249 249 249 F F F 8 160-179 138 138 138 249 249 249 F F F 9 180-199 138 138 138 249 249 249 F F F 10 200-219 139 139 139 252 252 252 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 137 137 137 248 248 248 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 139 139 139 252 252 252 F F F 19 380-399 134 134 134 242 242 242 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 136 136 136 245 245 245 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 F F F 27 540-559 138 138 138 250 250 250 F F F 28 560-579 137 137 137 248 248 248 F F F 29 580-599 135 135 135 244 244 244 F F F Max gen: 139, Max flush: 252 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.932781681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.977420603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.978162138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.978205022Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.978219483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.978240386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750} {"msg":"Crucible 98b8c460-5f77-42de-a63e-8d23289a8438 has session id: 9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.978851787Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Upstairs opts: Upstairs UUID: 98b8c460-5f77-42de-a63e-8d23289a8438, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.97887981Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Crucible stats registered with UUID: 98b8c460-5f77-42de-a63e-8d23289a8438","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.978899722Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:46.978913453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979392902Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979415564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750} {"msg":"98b8c460-5f77-42de-a63e-8d23289a8438 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979492862Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979509934Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"0","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979533886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"0","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979554008Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"1","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979580161Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"1","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979603383Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"2","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979629646Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"2","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979652318Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979677761Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979702893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51991)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.97976921Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48263)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979803273Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:36231)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.979827546Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.980516705Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"1","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.980559059Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"0","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.981265181Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"1","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.981298024Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"0","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.981861431Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.981986154Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.982020077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"2","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.98254953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"2","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[0]R flush_numbers[0..12]: [251, 252, 251, 227, 245, 245, 227, 249, 249, 249, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983063912Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[0]R generation[0..12]: [138, 139, 138, 129, 136, 136, 129, 138, 138, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983087754Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983102696Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[1]R flush_numbers[0..12]: [251, 252, 251, 227, 245, 245, 227, 249, 249, 249, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983117828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[1]R generation[0..12]: [138, 139, 138, 129, 136, 136, 129, 138, 138, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983135069Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983149201Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[2]R flush_numbers[0..12]: [251, 252, 251, 227, 245, 245, 227, 249, 249, 249, 252, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983162502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[2]R generation[0..12]: [138, 139, 138, 129, 136, 136, 129, 138, 138, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983175643Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983189195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983201966Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983214957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.98323598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983248741Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"":"downstairs","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983261872Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983273503Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"0","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983286775Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"1","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983299716Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"client":"2","":"downstairs","session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983312877Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983324738Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"98b8c460-5f77-42de-a63e-8d23289a8438 is now active with session: 9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.98333948Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983351571Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750,"session_id":"9d9f6c5e-fb39-46a8-88c6-e580e53ee1fb"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:36:46.983363492Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 204 len 4096 data: 7 02/30 Read block 434 len 24576 03/30 Read block 449 len 32768 04/30 Read block 179 len 20480 05/30 Read block 569 len 28672 06/30 Write block 549 len 16384 data: 20 20 16 14 07/30 Write block 370 len 24576 data: 11 14 13 10 10 7 08/30 Read block 98 len 4096 09/30 Read block 287 len 8192 10/30 Read block 413 len 32768 11/30 Read block 73 len 16384 12/30 Read block 21 len 32768 13/30 Read block 566 len 16384 14/30 Write block 26 len 20480 data: 9 13 11 11 9 15/30 Write block 69 len 28672 data: 10 11 12 10 10 11 12 16/30 Read block 468 len 4096 17/30 Write block 537 len 32768 data: 5 9 8 8 11 13 13 14 18/30 Read block 485 len 40960 19/30 Write block 156 len 20480 data: 9 10 10 10 9 20/30 Read block 440 len 8192 21/30 Write block 14 len 8192 data: 16 18 22/30 Write block 50 len 8192 data: 8 10 23/30 Read block 477 len 20480 24/30 Read block 309 len 32768 25/30 Read block 361 len 20480 26/30 Write block 573 len 28672 data: 14 12 13 10 10 10 9 27/30 Read block 379 len 4096 28/30 Read block 152 len 40960 29/30 Read block 122 len 28672 30/30 Read block 377 len 24576 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:36:47.003583122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 251 251 251 T T T 1 020-039 139 139 139 252 252 252 T T T 2 040-059 138 138 138 251 251 251 T T T 3 060-079 129 129 129 227 227 227 T T T 4 080-099 136 136 136 245 245 245 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 138 138 138 249 249 249 T T T 8 160-179 138 138 138 249 249 249 T T T 9 180-199 138 138 138 249 249 249 F F F 10 200-219 139 139 139 252 252 252 T T T 11 220-239 139 139 139 252 252 252 F F F 12 240-259 137 137 137 248 248 248 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 139 139 139 252 252 252 T T T 19 380-399 134 134 134 242 242 242 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 136 136 136 245 245 245 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 127 127 127 225 225 225 T T T 27 540-559 138 138 138 250 250 250 T T T 28 560-579 137 137 137 248 248 248 T T T 29 580-599 135 135 135 244 244 244 F F F Max gen: 139, Max flush: 252 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 140 140 140 253 253 253 F F F 2 040-059 140 140 140 253 253 253 F F F 3 060-079 140 140 140 253 253 253 F F F 4 080-099 136 136 136 245 245 245 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 140 140 140 253 253 253 F F F 9 180-199 138 138 138 249 249 249 F F F 10 200-219 140 140 140 253 253 253 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 137 137 137 248 248 248 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 140 140 140 253 253 253 F F F 19 380-399 134 134 134 242 242 242 F F F 20 400-419 139 139 139 252 252 252 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 136 136 136 245 245 245 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 F F F 26 520-539 140 140 140 253 253 253 F F F 27 540-559 140 140 140 253 253 253 F F F 28 560-579 140 140 140 253 253 253 F F F 29 580-599 135 135 135 244 244 244 F F F Max gen: 140, Max flush: 253 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.277881155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.322456901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.323287175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.32334496Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.323360802Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.323374963Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758} {"msg":"Crucible c891301b-964a-496c-b1e2-108a5f282333 has session id: 930b4c08-ee4f-4321-aeed-dee0083d7dcb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.323903477Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Upstairs opts: Upstairs UUID: c891301b-964a-496c-b1e2-108a5f282333, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.323941461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Crucible stats registered with UUID: c891301b-964a-496c-b1e2-108a5f282333","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.323963103Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:01.323976774Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324381625Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324413678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758} {"msg":"c891301b-964a-496c-b1e2-108a5f282333 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324467803Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324493906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"0","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.3245323Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"0","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324559873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324602927Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.3246284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"2","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324654132Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"2","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324679925Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324704778Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.32472761Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:64752)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324752042Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48167)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.324778855Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:01.324807068Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:01.32483408Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:01.325056093Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.325208938Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.325662574Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"2","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.325701308Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"0","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.326420671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"0","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.326456624Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"2","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.32721096Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:01.327302889Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.326238202Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34811)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.326376576Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.327488018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328176537Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[0]R flush_numbers[0..12]: [253, 253, 253, 253, 245, 245, 227, 253, 253, 249, 253, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328764527Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[0]R generation[0..12]: [140, 140, 140, 140, 136, 136, 129, 140, 140, 138, 140, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328789599Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328805541Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[1]R flush_numbers[0..12]: [253, 253, 253, 253, 245, 245, 227, 253, 253, 249, 253, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328819662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[1]R generation[0..12]: [140, 140, 140, 140, 136, 136, 129, 140, 140, 138, 140, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328838744Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328853276Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[2]R flush_numbers[0..12]: [253, 253, 253, 253, 245, 245, 227, 253, 253, 249, 253, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328866867Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[2]R generation[0..12]: [140, 140, 140, 140, 136, 136, 129, 140, 140, 138, 140, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328880068Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.32889362Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Max found gen is 141","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328906721Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Generation requested: 142 >= found:141","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328919402Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328931854Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328950516Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"":"downstairs","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328963257Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328974838Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"0","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.328987609Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"1","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.32900037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"client":"2","":"downstairs","session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.329013102Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.329024963Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"c891301b-964a-496c-b1e2-108a5f282333 is now active with session: 930b4c08-ee4f-4321-aeed-dee0083d7dcb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.329039714Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.329051486Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758,"session_id":"930b4c08-ee4f-4321-aeed-dee0083d7dcb"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:11.329063307Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 199 len 32768 data: 12 11 11 12 9 8 9 8 02/30 Read block 273 len 40960 03/30 Write block 567 len 4096 data: 11 04/30 Write block 521 len 8192 data: 12 11 05/30 Read block 34 len 36864 06/30 Flush 07/30 Write block 80 len 36864 data: 11 9 9 10 10 12 15 15 15 08/30 Read block 108 len 32768 09/30 Flush 10/30 Write block 412 len 16384 data: 14 14 13 13 11/30 Write block 443 len 24576 data: 7 7 7 11 9 9 12/30 Read block 163 len 8192 13/30 Read block 98 len 4096 14/30 Read block 541 len 20480 15/30 Read block 29 len 36864 16/30 Write block 45 len 8192 data: 9 10 17/30 Read block 552 len 16384 18/30 Read block 428 len 8192 19/30 Read block 32 len 20480 20/30 Write block 163 len 28672 data: 12 10 10 9 10 10 9 21/30 Read block 271 len 28672 22/30 Read block 215 len 36864 23/30 Flush 24/30 Flush 25/30 Write block 509 len 28672 data: 11 11 8 10 11 14 13 26/30 Read block 583 len 12288 27/30 Read block 24 len 32768 28/30 Read block 463 len 12288 29/30 Read block 481 len 24576 30/30 Write block 366 len 32768 data: 10 11 12 13 12 15 14 11 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:11.35031317Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 140 140 140 253 253 253 F F F 2 040-059 142 142 142 256 256 256 F F F 3 060-079 140 140 140 253 253 253 F F F 4 080-099 142 142 142 255 255 255 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 142 142 142 254 254 254 F F F 10 200-219 142 142 142 254 254 254 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 137 137 137 248 248 248 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 140 140 140 253 253 253 T T T 19 380-399 134 134 134 242 242 242 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 131 131 131 235 235 235 T T T 26 520-539 142 142 142 254 254 254 F F F 27 540-559 140 140 140 253 253 253 F F F 28 560-579 142 142 142 254 254 254 F F F 29 580-599 135 135 135 244 244 244 F F F Max gen: 142, Max flush: 256 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 140 140 140 253 253 253 F F F 2 040-059 142 142 142 256 256 256 F F F 3 060-079 140 140 140 253 253 253 F F F 4 080-099 142 142 142 255 255 255 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 142 142 142 254 254 254 F F F 10 200-219 142 142 142 254 254 254 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 137 137 137 248 248 248 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 143 143 143 257 257 257 F F F 19 380-399 134 134 134 242 242 242 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 130 130 130 230 230 230 F F F 25 500-519 143 143 143 257 257 257 F F F 26 520-539 142 142 142 254 254 254 F F F 27 540-559 140 140 140 253 253 253 F F F 28 560-579 142 142 142 254 254 254 F F F 29 580-599 135 135 135 244 244 244 F F F Max gen: 143, Max flush: 257 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.584551412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:25.627902264Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.628331197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.660531815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.661302933Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.661341277Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.661361369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.661379141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766} {"msg":"Crucible 03bbb46b-a741-4b16-aa5e-54dd1c63b691 has session id: c8bb41c7-1f72-4423-a23a-84fdeb47b992","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662000953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Upstairs opts: Upstairs UUID: 03bbb46b-a741-4b16-aa5e-54dd1c63b691, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662034956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Crucible stats registered with UUID: 03bbb46b-a741-4b16-aa5e-54dd1c63b691","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662053118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:25.66206852Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662485692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662509615Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766} {"msg":"03bbb46b-a741-4b16-aa5e-54dd1c63b691 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662600184Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662618935Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"0","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662645108Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"0","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.66265996Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"1","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662674031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"1","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662690883Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"2","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662705354Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"2","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662718125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662731457Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662744618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"ds_connection connected from Ok(127.0.0.1:48454)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.662881482Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"ds_connection connected from Ok(127.0.0.1:43528)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.663143628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"ds_connection connected from Ok(127.0.0.1:59498)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.663182362Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.663738759Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"2","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.663765611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"1","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.663984093Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"0","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.664428978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"1","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.66454574Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"2","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.664565902Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"0","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665148321Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665256401Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[0]R flush_numbers[0..12]: [253, 253, 256, 253, 255, 245, 227, 253, 256, 254, 254, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665321048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[0]R generation[0..12]: [140, 140, 142, 140, 142, 136, 129, 140, 142, 142, 142, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665348121Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665363902Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[1]R flush_numbers[0..12]: [253, 253, 256, 253, 255, 245, 227, 253, 256, 254, 254, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665378264Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[1]R generation[0..12]: [140, 140, 142, 140, 142, 136, 129, 140, 142, 142, 142, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665392645Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665406276Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[2]R flush_numbers[0..12]: [253, 253, 256, 253, 255, 245, 227, 253, 256, 254, 254, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665419828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[2]R generation[0..12]: [140, 140, 142, 140, 142, 136, 129, 140, 142, 142, 142, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665432959Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665446391Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665458762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665472573Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:37:25.665485175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665506057Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"":"downstairs","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665521848Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.6655338Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"0","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665546641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"1","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665559292Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"client":"2","":"downstairs","session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665571963Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665583725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"03bbb46b-a741-4b16-aa5e-54dd1c63b691 is now active with session: c8bb41c7-1f72-4423-a23a-84fdeb47b992","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665596266Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665607667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766,"session_id":"c8bb41c7-1f72-4423-a23a-84fdeb47b992"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:25.665619378Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 80 len 36864 data: 12 10 10 11 11 13 16 16 16 02/30 Read block 212 len 12288 03/30 Read block 506 len 20480 04/30 Write block 3 len 32768 data: 11 11 12 13 13 15 13 14 05/30 Read block 237 len 32768 06/30 Write block 190 len 32768 data: 4 7 9 12 13 15 11 11 07/30 Read block 263 len 32768 08/30 Read block 251 len 32768 09/30 Read block 360 len 8192 10/30 Write block 485 len 36864 data: 8 8 10 13 11 9 10 9 9 11/30 Write block 522 len 8192 data: 12 9 12/30 Read block 471 len 8192 13/30 Flush 14/30 Write block 64 len 4096 data: 10 15/30 Read block 24 len 16384 16/30 Read block 438 len 4096 17/30 Write block 519 len 32768 data: 14 12 13 13 10 10 11 13 18/30 Flush 19/30 Write block 582 len 40960 data: 9 9 10 8 8 8 9 9 11 12 20/30 Write block 146 len 28672 data: 8 10 8 13 11 12 11 21/30 Write block 10 len 4096 data: 15 22/30 Read block 25 len 36864 23/30 Read block 360 len 40960 24/30 Write block 49 len 32768 data: 8 9 11 10 9 13 14 13 25/30 Read block 149 len 8192 26/30 Write block 153 len 28672 data: 12 11 10 10 11 11 11 27/30 Read block 561 len 40960 28/30 Read block 120 len 28672 29/30 Read block 212 len 24576 30/30 Read block 514 len 36864 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:25.688214707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 258 258 258 T T T 1 020-039 140 140 140 253 253 253 F F F 2 040-059 142 142 142 256 256 256 T T T 3 060-079 144 144 144 259 259 259 F F F 4 080-099 144 144 144 258 258 258 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 140 140 140 253 253 253 T T T 8 160-179 142 142 142 256 256 256 F F F 9 180-199 144 144 144 258 258 258 F F F 10 200-219 142 142 142 254 254 254 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 137 137 137 248 248 248 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 143 143 143 257 257 257 F F F 19 380-399 134 134 134 242 242 242 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 144 144 144 258 258 258 F F F 25 500-519 144 144 144 259 259 259 F F F 26 520-539 144 144 144 259 259 259 F F F 27 540-559 140 140 140 253 253 253 F F F 28 560-579 142 142 142 254 254 254 F F F 29 580-599 135 135 135 244 244 244 T T T Max gen: 144, Max flush: 259 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 260 260 260 F F F 1 020-039 140 140 140 253 253 253 F F F 2 040-059 145 145 145 260 260 260 F F F 3 060-079 144 144 144 259 259 259 F F F 4 080-099 144 144 144 258 258 258 F F F 5 100-119 136 136 136 245 245 245 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 145 145 145 260 260 260 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 144 144 144 258 258 258 F F F 10 200-219 142 142 142 254 254 254 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 137 137 137 248 248 248 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 128 128 128 226 226 226 F F F 17 340-359 136 136 136 247 247 247 F F F 18 360-379 143 143 143 257 257 257 F F F 19 380-399 134 134 134 242 242 242 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 139 139 139 252 252 252 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 144 144 144 258 258 258 F F F 25 500-519 144 144 144 259 259 259 F F F 26 520-539 144 144 144 259 259 259 F F F 27 540-559 140 140 140 253 253 253 F F F 28 560-579 142 142 142 254 254 254 F F F 29 580-599 145 145 145 260 260 260 F F F Max gen: 145, Max flush: 260 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:29.962574356Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.008209228Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.009099378Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.009149263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.009167275Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.009184987Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774} {"msg":"Crucible e9d1ce00-3adf-4719-bc0f-5db053e3f04d has session id: e41f7b4c-a274-41ed-ab0f-33c1588253a7","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.009726121Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Upstairs opts: Upstairs UUID: e9d1ce00-3adf-4719-bc0f-5db053e3f04d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.009770246Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Crucible stats registered with UUID: e9d1ce00-3adf-4719-bc0f-5db053e3f04d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.009798599Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:30.009821221Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010463266Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010494029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774} {"msg":"e9d1ce00-3adf-4719-bc0f-5db053e3f04d active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010593829Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010611261Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"0","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010633643Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"0","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010646974Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"1","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010660996Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"1","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010673857Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"2","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010687928Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"2","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.01070106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010714831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010728022Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49527)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.010961046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62349)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.01099711Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34022)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.011080978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.011651665Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"2","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.01169102Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"0","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.012335505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"0","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.012505192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"1","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.012537585Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"2","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.012916813Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013154767Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013232125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"1","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[0]R flush_numbers[0..12]: [260, 253, 260, 259, 258, 245, 227, 260, 256, 258, 254, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013752797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[0]R generation[0..12]: [145, 140, 145, 144, 144, 136, 129, 145, 142, 144, 142, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.01377388Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013788911Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[1]R flush_numbers[0..12]: [260, 253, 260, 259, 258, 245, 227, 260, 256, 258, 254, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013802782Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[1]R generation[0..12]: [145, 140, 145, 144, 144, 136, 129, 145, 142, 144, 142, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013816624Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013830345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[2]R flush_numbers[0..12]: [260, 253, 260, 259, 258, 245, 227, 260, 256, 258, 254, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013844517Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[2]R generation[0..12]: [145, 140, 145, 144, 144, 136, 129, 145, 142, 144, 142, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013861248Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.01387543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013894262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013907603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013920344Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013932826Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"":"downstairs","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013945577Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013957168Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"0","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013969969Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"1","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013982731Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"client":"2","":"downstairs","session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.013995502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.014007563Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"e9d1ce00-3adf-4719-bc0f-5db053e3f04d is now active with session: e41f7b4c-a274-41ed-ab0f-33c1588253a7","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:37:30.014020144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.014035716Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774,"session_id":"e41f7b4c-a274-41ed-ab0f-33c1588253a7"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:30.014047497Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 391 len 12288 data: 8 10 10 02/30 Write block 331 len 36864 data: 8 13 10 13 12 13 9 9 7 03/30 Read block 276 len 24576 04/30 Flush 05/30 Write block 421 len 8192 data: 12 13 06/30 Write block 66 len 20480 data: 8 9 10 11 12 07/30 Write block 210 len 28672 data: 11 12 13 14 12 16 17 08/30 Read block 382 len 28672 09/30 Read block 108 len 20480 10/30 Write block 38 len 8192 data: 10 11 11/30 Write block 529 len 32768 data: 15 13 13 9 7 6 5 5 12/30 Flush 13/30 Write block 194 len 36864 data: 14 16 12 12 9 13 12 12 13 14/30 Read block 589 len 4096 15/30 Read block 151 len 32768 16/30 Write block 537 len 24576 data: 6 10 9 9 12 14 17/30 Write block 375 len 20480 data: 8 7 9 10 9 18/30 Read block 410 len 4096 19/30 Read block 26 len 28672 20/30 Read block 111 len 4096 21/30 Read block 350 len 36864 22/30 Read block 550 len 24576 23/30 Read block 270 len 28672 24/30 Write block 344 len 40960 data: 4 3 3 6 8 8 10 8 11 10 25/30 Write block 103 len 28672 data: 8 6 8 9 9 11 14 26/30 Read block 101 len 32768 27/30 Read block 174 len 32768 28/30 Write block 246 len 8192 data: 12 11 29/30 Write block 61 len 32768 data: 12 10 11 11 9 9 10 11 30/30 Read block 478 len 4096 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 9 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 17 18 16 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:30.034145844Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 260 260 260 F F F 1 020-039 146 146 146 262 262 262 F F F 2 040-059 145 145 145 260 260 260 F F F 3 060-079 146 146 146 262 262 262 T T T 4 080-099 144 144 144 258 258 258 F F F 5 100-119 136 136 136 245 245 245 T T T 6 120-139 129 129 129 227 227 227 F F F 7 140-159 145 145 145 260 260 260 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 144 144 144 258 258 258 T T T 10 200-219 146 146 146 262 262 262 T T T 11 220-239 139 139 139 252 252 252 F F F 12 240-259 137 137 137 248 248 248 T T T 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 136 136 136 247 247 247 T T T 18 360-379 143 143 143 257 257 257 T T T 19 380-399 146 146 146 261 261 261 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 146 146 146 262 262 262 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 144 144 144 258 258 258 F F F 25 500-519 144 144 144 259 259 259 F F F 26 520-539 146 146 146 262 262 262 T T T 27 540-559 140 140 140 253 253 253 T T T 28 560-579 142 142 142 254 254 254 F F F 29 580-599 145 145 145 260 260 260 F F F Max gen: 146, Max flush: 262 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 260 260 260 F F F 1 020-039 146 146 146 262 262 262 F F F 2 040-059 145 145 145 260 260 260 F F F 3 060-079 147 147 147 263 263 263 F F F 4 080-099 144 144 144 258 258 258 F F F 5 100-119 147 147 147 263 263 263 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 145 145 145 260 260 260 F F F 8 160-179 142 142 142 256 256 256 F F F 9 180-199 147 147 147 263 263 263 F F F 10 200-219 147 147 147 263 263 263 F F F 11 220-239 139 139 139 252 252 252 F F F 12 240-259 147 147 147 263 263 263 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 134 134 134 242 242 242 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 147 147 147 263 263 263 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 146 146 146 261 261 261 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 146 146 146 262 262 262 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 144 144 144 258 258 258 F F F 25 500-519 144 144 144 259 259 259 F F F 26 520-539 147 147 147 263 263 263 F F F 27 540-559 147 147 147 263 263 263 F F F 28 560-579 142 142 142 254 254 254 F F F 29 580-599 145 145 145 260 260 260 F F F Max gen: 147, Max flush: 263 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.317726592Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.362006669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.362835042Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.362886507Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.362907819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.362922791Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782} {"msg":"Crucible 3b0f833b-42d0-495e-84f6-8a383afcc4f2 has session id: 6830dbc9-b380-4b79-b2ea-d8007f052f0f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.36351431Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Upstairs opts: Upstairs UUID: 3b0f833b-42d0-495e-84f6-8a383afcc4f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.363551894Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Crucible stats registered with UUID: 3b0f833b-42d0-495e-84f6-8a383afcc4f2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.363569536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:44.363582407Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364012471Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364038663Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782} {"msg":"3b0f833b-42d0-495e-84f6-8a383afcc4f2 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364115081Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364131323Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"0","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364169236Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"0","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364184458Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"1","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.36419863Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"1","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364221522Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364251405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364272207Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364291419Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364313231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:44.364422932Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51700)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364457125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59739)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364481368Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:44.364507121Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:44.364535073Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.364559046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.365057626Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"0","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.365086069Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"1","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.365606741Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"1","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.365718383Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"0","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.366231404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:44.366320333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.366328134Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54316)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.366470098Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.367468298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368083111Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[0]R flush_numbers[0..12]: [260, 262, 260, 263, 258, 263, 227, 260, 256, 263, 263, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368723085Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[0]R generation[0..12]: [145, 146, 145, 147, 144, 147, 129, 145, 142, 147, 147, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368752038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.36877236Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[1]R flush_numbers[0..12]: [260, 262, 260, 263, 258, 263, 227, 260, 256, 263, 263, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368789982Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[1]R generation[0..12]: [145, 146, 145, 147, 144, 147, 129, 145, 142, 147, 147, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368807364Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368825045Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[2]R flush_numbers[0..12]: [260, 262, 260, 263, 258, 263, 227, 260, 256, 263, 263, 252]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368842127Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[2]R generation[0..12]: [145, 146, 145, 147, 144, 147, 129, 145, 142, 147, 147, 139]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368858859Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368884611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368900923Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368916815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368932746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368948598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"":"downstairs","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368964389Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.368996452Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"0","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.369013784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"1","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.369029546Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"client":"2","":"downstairs","session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.369045237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.369059969Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"3b0f833b-42d0-495e-84f6-8a383afcc4f2 is now active with session: 6830dbc9-b380-4b79-b2ea-d8007f052f0f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.369075251Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.369093682Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782,"session_id":"6830dbc9-b380-4b79-b2ea-d8007f052f0f"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:54.369108804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 429 len 40960 data: 11 12 13 10 7 7 6 8 7 9 02/30 Write block 308 len 36864 data: 12 13 13 12 12 9 8 11 11 03/30 Read block 461 len 40960 04/30 Write block 301 len 36864 data: 16 18 17 15 12 14 12 13 14 05/30 Write block 203 len 32768 data: 10 9 10 9 8 8 10 12 06/30 Flush 07/30 Write block 345 len 28672 data: 4 4 7 9 9 11 9 08/30 Read block 77 len 28672 09/30 Flush 10/30 Write block 389 len 16384 data: 9 7 9 11 11/30 Flush 12/30 Write block 111 len 28672 data: 12 12 10 8 7 11 13 13/30 Read block 415 len 20480 14/30 Write block 181 len 36864 data: 9 8 6 10 11 10 7 6 5 15/30 Write block 226 len 20480 data: 16 13 14 12 14 16/30 Read block 584 len 12288 17/30 Read block 207 len 40960 18/30 Flush 19/30 Write block 482 len 16384 data: 8 6 7 9 20/30 Write block 505 len 8192 data: 9 9 21/30 Read block 76 len 24576 22/30 Read block 131 len 28672 23/30 Read block 141 len 24576 24/30 Write block 46 len 28672 data: 11 9 7 9 10 12 11 25/30 Read block 107 len 12288 26/30 Write block 228 len 8192 data: 15 13 27/30 Read block 445 len 12288 28/30 Read block 182 len 28672 29/30 Write block 169 len 20480 data: 10 10 8 8 8 30/30 Read block 582 len 24576 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:54.391404623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 260 260 260 F F F 1 020-039 146 146 146 262 262 262 F F F 2 040-059 145 145 145 260 260 260 T T T 3 060-079 147 147 147 263 263 263 F F F 4 080-099 144 144 144 258 258 258 F F F 5 100-119 148 148 148 267 267 267 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 145 145 145 260 260 260 F F F 8 160-179 142 142 142 256 256 256 T T T 9 180-199 148 148 148 267 267 267 F F F 10 200-219 148 148 148 264 264 264 F F F 11 220-239 148 148 148 267 267 267 T T T 12 240-259 147 147 147 263 263 263 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 148 148 148 264 264 264 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 148 148 148 265 265 265 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 148 148 148 266 266 266 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 148 148 148 264 264 264 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 144 144 144 258 258 258 T T T 25 500-519 144 144 144 259 259 259 T T T 26 520-539 147 147 147 263 263 263 F F F 27 540-559 147 147 147 263 263 263 F F F 28 560-579 142 142 142 254 254 254 F F F 29 580-599 145 145 145 260 260 260 F F F Max gen: 148, Max flush: 267 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 260 260 260 F F F 1 020-039 146 146 146 262 262 262 F F F 2 040-059 149 149 149 268 268 268 F F F 3 060-079 147 147 147 263 263 263 F F F 4 080-099 144 144 144 258 258 258 F F F 5 100-119 148 148 148 267 267 267 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 145 145 145 260 260 260 F F F 8 160-179 149 149 149 268 268 268 F F F 9 180-199 148 148 148 267 267 267 F F F 10 200-219 148 148 148 264 264 264 F F F 11 220-239 149 149 149 268 268 268 F F F 12 240-259 147 147 147 263 263 263 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 148 148 148 264 264 264 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 148 148 148 265 265 265 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 148 148 148 266 266 266 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 148 148 148 264 264 264 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 149 149 149 268 268 268 F F F 25 500-519 149 149 149 268 268 268 F F F 26 520-539 147 147 147 263 263 263 F F F 27 540-559 147 147 147 263 263 263 F F F 28 560-579 142 142 142 254 254 254 F F F 29 580-599 145 145 145 260 260 260 F F F Max gen: 149, Max flush: 268 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.63213174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.678716908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.679391496Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.67943395Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.679448472Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.679463584Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790} {"msg":"Crucible e5c6dc9f-7a96-4d1b-b158-eef0977aaeac has session id: f1d8e68d-3974-4160-b485-33a59902f3ae","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680063004Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Upstairs opts: Upstairs UUID: e5c6dc9f-7a96-4d1b-b158-eef0977aaeac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680095737Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Crucible stats registered with UUID: e5c6dc9f-7a96-4d1b-b158-eef0977aaeac","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680110789Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:58.68012325Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680666115Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.6807232Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790} {"msg":"e5c6dc9f-7a96-4d1b-b158-eef0977aaeac active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680793958Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680811909Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"0","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680835792Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"0","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680850213Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"1","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680864185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"1","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680876886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680890537Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680903169Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680925561Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.680953674Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:58.681057444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:58.681108569Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:37:58.681143463Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:44323)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.681169965Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:53476)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.681196698Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.681221661Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.68201069Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"0","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.682039833Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"1","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.682675457Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"0","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.68270281Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"1","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.683270197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:37:58.683355166Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.682449563Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"ds_connection connected from Ok(127.0.0.1:42425)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.682533212Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.68370943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.684351555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[0]R flush_numbers[0..12]: [260, 262, 268, 263, 258, 267, 227, 260, 268, 267, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685010502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[0]R generation[0..12]: [145, 146, 149, 147, 144, 148, 129, 145, 149, 148, 148, 149]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685035314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685050656Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[1]R flush_numbers[0..12]: [260, 262, 268, 263, 258, 267, 227, 260, 268, 267, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685064107Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[1]R generation[0..12]: [145, 146, 149, 147, 144, 148, 129, 145, 149, 148, 148, 149]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685077899Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.68509203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[2]R flush_numbers[0..12]: [260, 262, 268, 263, 258, 267, 227, 260, 268, 267, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685105711Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[2]R generation[0..12]: [145, 146, 149, 147, 144, 148, 129, 145, 149, 148, 148, 149]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685119303Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685132934Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685145555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685162487Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Next flush: 269","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685181369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-02-27T17:38:08.685197491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"":"downstairs","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685219253Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685233724Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"0","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685249956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"1","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685265827Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"client":"2","":"downstairs","session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685281909Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685296711Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"e5c6dc9f-7a96-4d1b-b158-eef0977aaeac is now active with session: f1d8e68d-3974-4160-b485-33a59902f3ae","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685312412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685326834Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790,"session_id":"f1d8e68d-3974-4160-b485-33a59902f3ae"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:08.685341655Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 502 len 12288 data: 10 9 7 02/30 Write block 171 len 20480 data: 9 9 9 11 11 03/30 Flush 04/30 Write block 86 len 24576 data: 17 17 17 16 15 12 05/30 Write block 14 len 32768 data: 17 19 16 13 11 11 9 8 06/30 Write block 590 len 32768 data: 12 13 10 11 9 6 5 4 07/30 Read block 417 len 28672 08/30 Write block 97 len 40960 data: 6 7 7 8 9 7 9 7 9 10 09/30 Read block 224 len 12288 10/30 Read block 44 len 32768 11/30 Read block 174 len 32768 12/30 Read block 289 len 36864 13/30 Read block 175 len 16384 14/30 Read block 592 len 20480 15/30 Flush 16/30 Write block 595 len 12288 data: 7 6 5 17/30 Read block 59 len 40960 18/30 Flush 19/30 Write block 86 len 8192 data: 18 18 20/30 Write block 30 len 40960 data: 10 12 12 12 11 11 11 12 11 12 21/30 Read block 405 len 16384 22/30 Read block 468 len 40960 23/30 Write block 425 len 40960 data: 11 13 12 11 12 13 14 11 8 8 24/30 Write block 572 len 32768 data: 15 15 13 14 11 11 11 10 25/30 Write block 150 len 36864 data: 12 13 12 13 12 11 11 12 12 26/30 Write block 535 len 40960 data: 6 6 7 11 10 10 13 15 14 15 27/30 Read block 211 len 24576 28/30 Read block 157 len 4096 29/30 Write block 250 len 36864 data: 17 17 18 22 18 12 12 11 11 30/30 Write block 356 len 12288 data: 12 14 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:08.70670433Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 270 270 270 F F F 1 020-039 150 150 150 270 270 270 T T T 2 040-059 149 149 149 268 268 268 F F F 3 060-079 147 147 147 263 263 263 F F F 4 080-099 150 150 150 270 270 270 T T T 5 100-119 150 150 150 270 270 270 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 145 145 145 260 260 260 T T T 8 160-179 150 150 150 269 269 269 F F F 9 180-199 148 148 148 267 267 267 F F F 10 200-219 148 148 148 264 264 264 F F F 11 220-239 149 149 149 268 268 268 F F F 12 240-259 147 147 147 263 263 263 T T T 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 148 148 148 264 264 264 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 148 148 148 265 265 265 T T T 18 360-379 147 147 147 263 263 263 F F F 19 380-399 148 148 148 266 266 266 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 148 148 148 264 264 264 T T T 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 149 149 149 268 268 268 F F F 25 500-519 150 150 150 269 269 269 F F F 26 520-539 147 147 147 263 263 263 T T T 27 540-559 147 147 147 263 263 263 T T T 28 560-579 142 142 142 254 254 254 T T T 29 580-599 150 150 150 271 271 271 F F F Max gen: 150, Max flush: 271 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 270 270 270 F F F 1 020-039 151 151 151 272 272 272 F F F 2 040-059 149 149 149 268 268 268 F F F 3 060-079 147 147 147 263 263 263 F F F 4 080-099 151 151 151 272 272 272 F F F 5 100-119 150 150 150 270 270 270 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 151 151 151 272 272 272 F F F 8 160-179 150 150 150 269 269 269 F F F 9 180-199 148 148 148 267 267 267 F F F 10 200-219 148 148 148 264 264 264 F F F 11 220-239 149 149 149 268 268 268 F F F 12 240-259 151 151 151 272 272 272 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 148 148 148 264 264 264 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 151 151 151 272 272 272 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 148 148 148 266 266 266 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 151 151 151 272 272 272 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 149 149 149 268 268 268 F F F 25 500-519 150 150 150 269 269 269 F F F 26 520-539 151 151 151 272 272 272 F F F 27 540-559 151 151 151 272 272 272 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 150 150 150 271 271 271 F F F Max gen: 151, Max flush: 272 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:12.97463117Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.019180893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.020069002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.020115417Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.020130368Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.02014483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798} {"msg":"Crucible 8407e2c8-9b16-40dc-b959-6b11227baf8b has session id: f6135941-458b-4ee8-a320-7d8a3b5c0c2a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.020717148Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Upstairs opts: Upstairs UUID: 8407e2c8-9b16-40dc-b959-6b11227baf8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.020760872Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Crucible stats registered with UUID: 8407e2c8-9b16-40dc-b959-6b11227baf8b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.020788925Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:13.020809567Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021327449Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021353722Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798} {"msg":"8407e2c8-9b16-40dc-b959-6b11227baf8b active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021425249Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021441781Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"0","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021466213Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"0","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021484255Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"1","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021511058Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"1","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.02153592Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"2","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021563333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"2","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021592846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.02173225Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021769324Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42278)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021797087Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44995)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021821279Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59681)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.021855973Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.022577385Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"2","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.022607088Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"0","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.02331613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"2","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.023457884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"0","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.02371647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"1","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.023919061Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.024053774Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.024358185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"1","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[0]R flush_numbers[0..12]: [270, 272, 268, 263, 272, 270, 227, 272, 269, 267, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.024926632Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[0]R generation[0..12]: [150, 151, 149, 147, 151, 150, 129, 151, 150, 148, 148, 149]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.024945274Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.024959376Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[1]R flush_numbers[0..12]: [270, 272, 268, 263, 272, 270, 227, 272, 269, 267, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.024973017Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[1]R generation[0..12]: [150, 151, 149, 147, 151, 150, 129, 151, 150, 148, 148, 149]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.024986478Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.02500406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[2]R flush_numbers[0..12]: [270, 272, 268, 263, 272, 270, 227, 272, 269, 267, 264, 268]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025018502Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[2]R generation[0..12]: [150, 151, 149, 147, 151, 150, 129, 151, 150, 148, 148, 149]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025031793Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025045564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Max found gen is 152","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025065976Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025079628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025092299Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.02510474Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"":"downstairs","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025117591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025129253Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"0","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025142024Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"1","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025154795Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"client":"2","":"downstairs","session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025167767Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025179688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"8407e2c8-9b16-40dc-b959-6b11227baf8b is now active with session: f6135941-458b-4ee8-a320-7d8a3b5c0c2a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.02519713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025210681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798,"session_id":"f6135941-458b-4ee8-a320-7d8a3b5c0c2a"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:13.025222712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 252 len 4096 data: 19 02/30 Write block 541 len 36864 data: 14 16 15 16 14 13 13 19 21 03/30 Read block 43 len 12288 04/30 Write block 593 len 4096 data: 12 05/30 Read block 474 len 24576 06/30 Read block 132 len 28672 07/30 Write block 219 len 20480 data: 14 13 13 14 16 08/30 Read block 584 len 16384 09/30 Flush 10/30 Write block 381 len 36864 data: 9 11 14 15 17 14 13 13 10 11/30 Flush 12/30 Write block 210 len 36864 data: 13 13 14 15 13 17 18 16 14 13/30 Read block 240 len 40960 14/30 Flush 15/30 Write block 309 len 16384 data: 15 14 13 13 16/30 Read block 278 len 8192 17/30 Write block 519 len 36864 data: 15 13 14 14 11 11 12 14 14 18/30 Read block 354 len 28672 19/30 Write block 323 len 16384 data: 18 17 15 14 20/30 Write block 246 len 24576 data: 13 12 12 16 18 18 21/30 Flush 22/30 Write block 62 len 32768 data: 11 12 12 10 10 11 12 12 23/30 Read block 175 len 4096 24/30 Read block 347 len 20480 25/30 Write block 163 len 28672 data: 13 11 11 10 11 11 11 26/30 Write block 152 len 32768 data: 13 14 13 12 12 13 13 12 27/30 Write block 420 len 28672 data: 12 13 14 12 9 12 14 28/30 Read block 386 len 40960 29/30 Read block 278 len 16384 30/30 Read block 488 len 12288 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:13.046270295Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 270 270 270 F F F 1 020-039 151 151 151 272 272 272 F F F 2 040-059 149 149 149 268 268 268 F F F 3 060-079 147 147 147 263 263 263 T T T 4 080-099 151 151 151 272 272 272 F F F 5 100-119 150 150 150 270 270 270 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 151 151 151 272 272 272 T T T 8 160-179 150 150 150 269 269 269 T T T 9 180-199 148 148 148 267 267 267 F F F 10 200-219 152 152 152 275 275 275 F F F 11 220-239 152 152 152 273 273 273 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 152 152 152 276 276 276 F F F 16 320-339 152 152 152 276 276 276 F F F 17 340-359 151 151 151 272 272 272 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 152 152 152 274 274 274 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 151 151 151 272 272 272 T T T 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 149 149 149 268 268 268 F F F 25 500-519 152 152 152 276 276 276 F F F 26 520-539 152 152 152 276 276 276 F F F 27 540-559 152 152 152 273 273 273 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 152, Max flush: 276 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 270 270 270 F F F 1 020-039 151 151 151 272 272 272 F F F 2 040-059 149 149 149 268 268 268 F F F 3 060-079 153 153 153 277 277 277 F F F 4 080-099 151 151 151 272 272 272 F F F 5 100-119 150 150 150 270 270 270 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 153 153 153 277 277 277 F F F 9 180-199 148 148 148 267 267 267 F F F 10 200-219 152 152 152 275 275 275 F F F 11 220-239 152 152 152 273 273 273 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 134 134 134 242 242 242 F F F 15 300-319 152 152 152 276 276 276 F F F 16 320-339 152 152 152 276 276 276 F F F 17 340-359 151 151 151 272 272 272 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 152 152 152 274 274 274 F F F 20 400-419 142 142 142 256 256 256 F F F 21 420-439 153 153 153 277 277 277 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 149 149 149 268 268 268 F F F 25 500-519 152 152 152 276 276 276 F F F 26 520-539 152 152 152 276 276 276 F F F 27 540-559 152 152 152 273 273 273 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 153, Max flush: 277 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.282290788Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.327019079Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.327788766Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.327838011Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.327862034Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.327880246Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806} {"msg":"Crucible 7dfaa4de-0621-4ba6-8232-ecbf5740f2a6 has session id: 18a4dbbe-af0a-41a8-bbcf-8bc3285e118f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.328463534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Upstairs opts: Upstairs UUID: 7dfaa4de-0621-4ba6-8232-ecbf5740f2a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.328501268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Crucible stats registered with UUID: 7dfaa4de-0621-4ba6-8232-ecbf5740f2a6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.328526731Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:17.328547273Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329025891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329058894Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806} {"msg":"7dfaa4de-0621-4ba6-8232-ecbf5740f2a6 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329133812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329150334Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"0","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329174026Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"0","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329187567Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329201609Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.32921456Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"2","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329228412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"2","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329244663Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329267206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329291838Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51391)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329597889Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:17.329644914Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37562)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329668856Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:17.329686348Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:17.32970908Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.329735493Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.330183058Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"0","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.33020943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"2","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.330773198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"2","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.330857186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"0","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.331312292Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:17.33139298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.330943254Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60868)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.331186809Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.332213913Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.332743456Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[0]R flush_numbers[0..12]: [270, 272, 268, 277, 272, 270, 227, 277, 277, 267, 275, 273]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333345797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[0]R generation[0..12]: [150, 151, 149, 153, 151, 150, 129, 153, 153, 148, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333365479Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.33338012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[1]R flush_numbers[0..12]: [270, 272, 268, 277, 272, 270, 227, 277, 277, 267, 275, 273]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333395272Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[1]R generation[0..12]: [150, 151, 149, 153, 151, 150, 129, 153, 153, 148, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333409263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333423084Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[2]R flush_numbers[0..12]: [270, 272, 268, 277, 272, 270, 227, 277, 277, 267, 275, 273]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333436636Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[2]R generation[0..12]: [150, 151, 149, 153, 151, 150, 129, 153, 153, 148, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333450937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333469989Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333482981Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333501852Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333514474Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333527145Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"":"downstairs","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333539776Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333551277Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"0","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333565589Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"1","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.33357856Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"client":"2","":"downstairs","session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333591681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333603633Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"7dfaa4de-0621-4ba6-8232-ecbf5740f2a6 is now active with session: 18a4dbbe-af0a-41a8-bbcf-8bc3285e118f","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333620374Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333632296Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806,"session_id":"18a4dbbe-af0a-41a8-bbcf-8bc3285e118f"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:27.333644337Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 382 len 40960 data: 12 15 16 18 15 14 14 11 8 10 02/30 Read block 342 len 16384 03/30 Read block 483 len 12288 04/30 Flush 05/30 Flush 06/30 Write block 25 len 4096 data: 9 07/30 Write block 234 len 8192 data: 13 16 08/30 Read block 232 len 32768 09/30 Flush 10/30 Write block 218 len 32768 data: 15 15 14 14 15 17 16 17 11/30 Write block 350 len 12288 data: 12 10 12 12/30 Read block 380 len 40960 13/30 Read block 405 len 4096 14/30 Write block 396 len 20480 data: 11 10 7 7 7 15/30 Read block 583 len 28672 16/30 Read block 149 len 36864 17/30 Write block 184 len 32768 data: 11 12 11 8 7 6 5 8 18/30 Write block 305 len 24576 data: 13 15 13 14 16 15 19/30 Write block 526 len 32768 data: 15 15 14 16 14 14 10 8 20/30 Read block 369 len 12288 21/30 Read block 508 len 24576 22/30 Write block 296 len 24576 data: 9 10 11 13 16 17 23/30 Read block 449 len 8192 24/30 Flush 25/30 Write block 520 len 32768 data: 14 15 15 12 12 13 16 16 26/30 Write block 342 len 40960 data: 8 9 5 5 5 8 10 10 13 11 27/30 Write block 501 len 20480 data: 10 11 10 8 10 28/30 Read block 174 len 16384 29/30 Write block 20 len 16384 data: 10 9 10 9 30/30 Read block 126 len 4096 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 5 6 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:27.353852915Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 270 270 270 F F F 1 020-039 154 154 154 280 280 280 T T T 2 040-059 149 149 149 268 268 268 F F F 3 060-079 153 153 153 277 277 277 F F F 4 080-099 151 151 151 272 272 272 F F F 5 100-119 150 150 150 270 270 270 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 153 153 153 277 277 277 F F F 9 180-199 154 154 154 281 281 281 F F F 10 200-219 154 154 154 281 281 281 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 152 152 152 276 276 276 F F F 17 340-359 154 154 154 281 281 281 T T T 18 360-379 147 147 147 263 263 263 F F F 19 380-399 154 154 154 281 281 281 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 153 153 153 277 277 277 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 149 149 149 268 268 268 F F F 25 500-519 152 152 152 276 276 276 T T T 26 520-539 154 154 154 281 281 281 T T T 27 540-559 152 152 152 273 273 273 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 154, Max flush: 281 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 270 270 270 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 149 149 149 268 268 268 F F F 3 060-079 153 153 153 277 277 277 F F F 4 080-099 151 151 151 272 272 272 F F F 5 100-119 150 150 150 270 270 270 F F F 6 120-139 129 129 129 227 227 227 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 153 153 153 277 277 277 F F F 9 180-199 154 154 154 281 281 281 F F F 10 200-219 154 154 154 281 281 281 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 152 152 152 276 276 276 F F F 13 260-279 139 139 139 252 252 252 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 152 152 152 276 276 276 F F F 17 340-359 155 155 155 282 282 282 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 154 154 154 281 281 281 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 153 153 153 277 277 277 F F F 22 440-459 142 142 142 256 256 256 F F F 23 460-479 138 138 138 249 249 249 F F F 24 480-499 149 149 149 268 268 268 F F F 25 500-519 155 155 155 282 282 282 F F F 26 520-539 155 155 155 282 282 282 F F F 27 540-559 152 152 152 273 273 273 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 155, Max flush: 282 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.590606831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:31.63362606Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.634069315Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.665859671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.666853141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.666887855Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.666901676Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.666915888Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814} {"msg":"Crucible cf1755f3-0876-40d6-ba50-83fb7c47e637 has session id: 7e65a008-e135-4a14-9491-858e6ccac2f8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.667558503Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Upstairs opts: Upstairs UUID: cf1755f3-0876-40d6-ba50-83fb7c47e637, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.667587106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Crucible stats registered with UUID: cf1755f3-0876-40d6-ba50-83fb7c47e637","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.667602187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:31.667614678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668120989Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668144032Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814} {"msg":"cf1755f3-0876-40d6-ba50-83fb7c47e637 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668218979Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668244652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"0","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668270184Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"0","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668284936Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"1","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668299297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"1","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668312459Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"2","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.66832814Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"2","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668351303Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668375725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668398597Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41344)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668493267Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37853)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668533601Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48850)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.668559433Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.669363445Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"1","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.669392188Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"2","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.669508689Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"0","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.670186538Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"1","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.670223081Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"2","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.670251854Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"0","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.670792349Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.67090322Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[0]R flush_numbers[0..12]: [270, 282, 268, 277, 272, 270, 227, 277, 277, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.670980538Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[0]R generation[0..12]: [150, 155, 149, 153, 151, 150, 129, 153, 153, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671012321Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671031773Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[1]R flush_numbers[0..12]: [270, 282, 268, 277, 272, 270, 227, 277, 277, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671045824Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[1]R generation[0..12]: [150, 155, 149, 153, 151, 150, 129, 153, 153, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671058966Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671072407Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[2]R flush_numbers[0..12]: [270, 282, 268, 277, 272, 270, 227, 277, 277, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671085688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[2]R generation[0..12]: [150, 155, 149, 153, 151, 150, 129, 153, 153, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.67109917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_idWait for a query_work_queue command to finish before sending IO ":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671119442Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671132083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671144804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671157266Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671169687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"":"downstairs","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671182188Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671193829Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"0","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.67120662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"1","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671219402Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"client":"2","":"downstairs","session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671234103Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671246175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"cf1755f3-0876-40d6-ba50-83fb7c47e637 is now active with session: 7e65a008-e135-4a14-9491-858e6ccac2f8","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671258406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671269917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814,"session_id":"7e65a008-e135-4a14-9491-858e6ccac2f8"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:31.671281688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 431 len 4096 data: 15 02/30 Read block 319 len 8192 03/30 Write block 93 len 32768 data: 11 8 6 6 7 8 8 9 04/30 Write block 334 len 16384 data: 14 13 14 10 05/30 Read block 319 len 36864 06/30 Read block 188 len 36864 07/30 Write block 544 len 36864 data: 17 15 14 14 20 22 21 17 15 08/30 Flush 09/30 Write block 473 len 40960 data: 8 7 7 9 6 5 3 5 6 9 10/30 Write block 4 len 4096 data: 12 11/30 Read block 443 len 20480 12/30 Read block 498 len 4096 13/30 Flush 14/30 Write block 120 len 24576 data: 19 19 16 15 14 12 15/30 Read block 228 len 36864 16/30 Read block 495 len 12288 17/30 Read block 124 len 32768 18/30 Read block 128 len 4096 19/30 Read block 141 len 12288 20/30 Write block 449 len 36864 data: 9 9 8 12 12 11 10 11 12 21/30 Read block 479 len 28672 22/30 Flush 23/30 Write block 257 len 24576 data: 12 12 13 12 10 9 24/30 Read block 141 len 32768 25/30 Read block 89 len 4096 26/30 Read block 167 len 28672 27/30 Write block 263 len 28672 data: 8 9 9 9 12 12 9 28/30 Read block 24 len 36864 29/30 Write block 468 len 20480 data: 15 14 14 15 11 30/30 Read block 448 len 8192 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:31.693084297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 149 149 149 268 268 268 F F F 3 060-079 153 153 153 277 277 277 F F F 4 080-099 156 156 156 283 283 283 F F F 5 100-119 156 156 156 283 283 283 F F F 6 120-139 156 156 156 285 285 285 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 153 153 153 277 277 277 F F F 9 180-199 154 154 154 281 281 281 F F F 10 200-219 154 154 154 281 281 281 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 152 152 152 276 276 276 T T T 13 260-279 139 139 139 252 252 252 T T T 14 280-299 154 154 154 281 281 281 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 156 156 156 283 283 283 F F F 17 340-359 155 155 155 282 282 282 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 154 154 154 281 281 281 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 156 156 156 285 285 285 F F F 23 460-479 156 156 156 284 284 284 T T T 24 480-499 156 156 156 284 284 284 F F F 25 500-519 155 155 155 282 282 282 F F F 26 520-539 155 155 155 282 282 282 F F F 27 540-559 156 156 156 283 283 283 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 156, Max flush: 285 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 149 149 149 268 268 268 F F F 3 060-079 153 153 153 277 277 277 F F F 4 080-099 156 156 156 283 283 283 F F F 5 100-119 156 156 156 283 283 283 F F F 6 120-139 156 156 156 285 285 285 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 153 153 153 277 277 277 F F F 9 180-199 154 154 154 281 281 281 F F F 10 200-219 154 154 154 281 281 281 F F F 11 220-239 154 154 154 281 281 281 F F F 12 240-259 157 157 157 286 286 286 F F F 13 260-279 157 157 157 286 286 286 F F F 14 280-299 154 154 154 281 281 281 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 156 156 156 283 283 283 F F F 17 340-359 155 155 155 282 282 282 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 154 154 154 281 281 281 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 156 156 156 283 283 283 F F F 22 440-459 156 156 156 285 285 285 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 156 156 156 284 284 284 F F F 25 500-519 155 155 155 282 282 282 F F F 26 520-539 155 155 155 282 282 282 F F F 27 540-559 156 156 156 283 283 283 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 157, Max flush: 286 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:35.959587352Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:36.00318869Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.003810123Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.035642623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.036464826Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.03650804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.036526482Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.036541144Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822} {"msg":"Crucible 17657642-1095-463b-aed9-2fe5e9e20f43 has session id: e99a0954-957d-45e6-bdb7-8f3192dae68f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.03710425Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Upstairs opts: Upstairs UUID: 17657642-1095-463b-aed9-2fe5e9e20f43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037142444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Crucible stats registered with UUID: 17657642-1095-463b-aed9-2fe5e9e20f43","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037158566Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:36.037172127Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037619302Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037641905Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822} {"msg":"17657642-1095-463b-aed9-2fe5e9e20f43 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037707901Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037724203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"0","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037745735Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"0","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037774538Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"1","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037802621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"1","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037829544Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"2","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037858777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"2","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037884039Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037912342Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037939485Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34303)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037968318Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53937)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.037997211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63521)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.038032444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.038775589Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"1","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.038816683Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"2","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.039102632Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"0","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.039636086Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"1","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.03967843Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"2","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.039739146Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"0","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040196693Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040309044Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[0]R flush_numbers[0..12]: [284, 282, 268, 277, 283, 283, 285, 277, 277, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040362889Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[0]R generation[0..12]: [156, 155, 149, 153, 156, 156, 156, 153, 153, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040381671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040396593Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[1]R flush_numbers[0..12]: [284, 282, 268, 277, 283, 283, 285, 277, 277, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040410294Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[1]R generation[0..12]: [156, 155, 149, 153, 156, 156, 156, 153, 153, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040435797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040450288Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[2]R flush_numbers[0..12]: [284, 282, 268, 277, 283, 283, 285, 277, 277, 281, 281, 281]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040463809Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[2]R generation[0..12]: [156, 155, 149, 153, 156, 156, 156, 153, 153, 154, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040477201Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040490952Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040507094Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:38:36.040520525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040543277Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.04056622Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"":"downstairs","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040584581Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040597343Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"0","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040610434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"1","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040623305Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"client":"2","":"downstairs","session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040645118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.04066458Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"17657642-1095-463b-aed9-2fe5e9e20f43 is now active with session: e99a0954-957d-45e6-bdb7-8f3192dae68f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040677741Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040689322Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822,"session_id":"e99a0954-957d-45e6-bdb7-8f3192dae68f"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:36.040701193Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 186 len 12288 data: 12 9 8 02/30 Flush 03/30 Flush 04/30 Write block 219 len 12288 data: 16 15 15 05/30 Read block 348 len 40960 06/30 Read block 425 len 16384 07/30 Read block 427 len 12288 08/30 Flush 09/30 Write block 532 len 40960 data: 11 9 7 7 7 8 12 11 11 15 10/30 Read block 27 len 8192 11/30 Write block 255 len 36864 data: 13 13 13 13 14 13 11 10 9 12/30 Read block 242 len 12288 13/30 Flush 14/30 Write block 499 len 32768 data: 8 7 11 12 11 9 11 10 15/30 Write block 124 len 12288 data: 15 13 9 16/30 Write block 218 len 36864 data: 16 17 16 16 16 18 17 18 17 17/30 Read block 390 len 28672 18/30 Read block 200 len 36864 19/30 Read block 190 len 24576 20/30 Read block 56 len 36864 21/30 Write block 426 len 4096 data: 15 22/30 Read block 455 len 24576 23/30 Flush 24/30 Write block 48 len 28672 data: 8 10 11 13 12 10 14 25/30 Flush 26/30 Flush 27/30 Write block 391 len 12288 data: 11 12 11 28/30 Read block 518 len 32768 29/30 Write block 291 len 32768 data: 13 12 10 9 9 10 11 12 30/30 Read block 92 len 28672 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:36.061604661Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 158 158 158 292 292 292 F F F 3 060-079 153 153 153 277 277 277 F F F 4 080-099 156 156 156 283 283 283 F F F 5 100-119 156 156 156 283 283 283 F F F 6 120-139 158 158 158 291 291 291 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 153 153 153 277 277 277 F F F 9 180-199 158 158 158 287 287 287 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 158 158 158 291 291 291 F F F 12 240-259 158 158 158 290 290 290 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 154 154 154 281 281 281 T T T 15 300-319 154 154 154 281 281 281 F F F 16 320-339 156 156 156 283 283 283 F F F 17 340-359 155 155 155 282 282 282 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 154 154 154 281 281 281 T T T 20 400-419 154 154 154 281 281 281 F F F 21 420-439 158 158 158 291 291 291 F F F 22 440-459 156 156 156 285 285 285 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 158 158 158 291 291 291 F F F 25 500-519 158 158 158 291 291 291 F F F 26 520-539 158 158 158 290 290 290 F F F 27 540-559 158 158 158 290 290 290 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 158, Max flush: 292 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 158 158 158 292 292 292 F F F 3 060-079 153 153 153 277 277 277 F F F 4 080-099 156 156 156 283 283 283 F F F 5 100-119 156 156 156 283 283 283 F F F 6 120-139 158 158 158 291 291 291 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 153 153 153 277 277 277 F F F 9 180-199 158 158 158 287 287 287 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 158 158 158 291 291 291 F F F 12 240-259 158 158 158 290 290 290 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 159 159 159 293 293 293 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 156 156 156 283 283 283 F F F 17 340-359 155 155 155 282 282 282 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 159 159 159 293 293 293 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 158 158 158 291 291 291 F F F 22 440-459 156 156 156 285 285 285 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 158 158 158 291 291 291 F F F 25 500-519 158 158 158 291 291 291 F F F 26 520-539 158 158 158 290 290 290 F F F 27 540-559 158 158 158 290 290 290 F F F 28 560-579 151 151 151 272 272 272 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 159, Max flush: 293 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.314476873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.359140538Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.360094485Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.360138649Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.36015309Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.360167762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830} {"msg":"Crucible 3659dbfd-ff7d-40a0-8b35-4622c712ec6b has session id: beed88f2-d89c-49ff-85c0-197b31e14045","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.360833209Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Upstairs opts: Upstairs UUID: 3659dbfd-ff7d-40a0-8b35-4622c712ec6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.360878273Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Crucible stats registered with UUID: 3659dbfd-ff7d-40a0-8b35-4622c712ec6b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.360901206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:40.360921078Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361416508Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361450001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830} {"msg":"3659dbfd-ff7d-40a0-8b35-4622c712ec6b active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361508017Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361532559Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"0","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361557812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"0","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361572043Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"1","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361586375Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"1","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361599536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"2","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361614258Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"2","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361682105Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361716498Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361740871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"ds_connection connected from Ok(127.0.0.1:51698)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361866023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"ds_connection connected from Ok(127.0.0.1:39275)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361919089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"ds_connection connected from Ok(127.0.0.1:33440)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.361952602Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.362612208Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"2","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.362656043Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"0","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.36332437Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"0","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.363487447Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"2","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.36391414Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.364064175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.364141653Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"1","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.364653614Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"1","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[0]R flush_numbers[0..12]: [284, 282, 292, 277, 283, 283, 291, 277, 277, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365211641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[0]R generation[0..12]: [156, 155, 158, 153, 156, 156, 158, 153, 153, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365231952Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365246434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[1]R flush_numbers[0..12]: [284, 282, 292, 277, 283, 283, 291, 277, 277, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365260235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[1]R generation[0..12]: [156, 155, 158, 153, 156, 156, 158, 153, 153, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365273647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365287378Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[2]R flush_numbers[0..12]: [284, 282, 292, 277, 283, 283, 291, 277, 277, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.36530461Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[2]R generation[0..12]: [156, 155, 158, 153, 156, 156, 158, 153, 153, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365318351Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365332123Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365344684Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365357625Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365370377Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.36540097Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"":"downstairs","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365428132Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365450885Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"0","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365473667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"1","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365498399Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"client":"2","":"downstairs","session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365522822Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365537743Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"3659dbfd-ff7d-40a0-8b35-4622c712ec6b is now active with session: beed88f2-d89c-49ff-85c0-197b31e14045","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365553665Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:40.365574507Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830,"session_id":"beed88f2-d89c-49ff-85c0-197b31e14045"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:38:40.3656013Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 71 len 20480 data: 13 11 11 12 13 02/30 Write block 76 len 20480 data: 15 10 10 12 13 03/30 Write block 175 len 4096 data: 12 04/30 Write block 76 len 20480 data: 16 11 11 13 14 05/30 Read block 108 len 32768 06/30 Flush 07/30 Write block 99 len 28672 data: 9 10 10 8 10 8 10 08/30 Flush 09/30 Flush 10/30 Flush 11/30 Write block 70 len 28672 data: 13 14 12 12 13 14 17 12/30 Read block 573 len 8192 13/30 Read block 287 len 32768 14/30 Read block 506 len 28672 15/30 Read block 347 len 20480 16/30 Read block 329 len 4096 17/30 Read block 539 len 32768 18/30 Read block 209 len 8192 19/30 Write block 534 len 32768 data: 8 8 8 9 13 12 12 16 20/30 Read block 348 len 8192 21/30 Read block 154 len 20480 22/30 Flush 23/30 Write block 528 len 24576 data: 15 17 15 15 12 10 24/30 Flush 25/30 Write block 443 len 12288 data: 8 8 8 26/30 Read block 269 len 28672 27/30 Write block 555 len 28672 data: 11 13 14 13 13 10 9 28/30 Write block 103 len 28672 data: 11 9 11 11 10 12 15 29/30 Write block 347 len 8192 data: 9 11 30/30 Read block 540 len 16384 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 3 4 Done 5 6 3 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:40.385307737Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 158 158 158 292 292 292 F F F 3 060-079 160 160 160 298 298 298 F F F 4 080-099 160 160 160 295 295 295 F F F 5 100-119 160 160 160 295 295 295 T T T 6 120-139 158 158 158 291 291 291 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 160 160 160 294 294 294 F F F 9 180-199 158 158 158 287 287 287 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 158 158 158 291 291 291 F F F 12 240-259 158 158 158 290 290 290 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 159 159 159 293 293 293 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 156 156 156 283 283 283 F F F 17 340-359 155 155 155 282 282 282 T T T 18 360-379 147 147 147 263 263 263 F F F 19 380-399 159 159 159 293 293 293 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 158 158 158 291 291 291 F F F 22 440-459 156 156 156 285 285 285 T T T 23 460-479 157 157 157 286 286 286 F F F 24 480-499 158 158 158 291 291 291 F F F 25 500-519 158 158 158 291 291 291 F F F 26 520-539 160 160 160 299 299 299 F F F 27 540-559 160 160 160 298 298 298 T T T 28 560-579 151 151 151 272 272 272 T T T 29 580-599 152 152 152 273 273 273 F F F Max gen: 160, Max flush: 299 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 158 158 158 292 292 292 F F F 3 060-079 160 160 160 298 298 298 F F F 4 080-099 160 160 160 295 295 295 F F F 5 100-119 161 161 161 300 300 300 F F F 6 120-139 158 158 158 291 291 291 F F F 7 140-159 153 153 153 277 277 277 F F F 8 160-179 160 160 160 294 294 294 F F F 9 180-199 158 158 158 287 287 287 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 158 158 158 291 291 291 F F F 12 240-259 158 158 158 290 290 290 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 159 159 159 293 293 293 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 156 156 156 283 283 283 F F F 17 340-359 161 161 161 300 300 300 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 159 159 159 293 293 293 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 158 158 158 291 291 291 F F F 22 440-459 161 161 161 300 300 300 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 158 158 158 291 291 291 F F F 25 500-519 158 158 158 291 291 291 F F F 26 520-539 160 160 160 299 299 299 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 161, Max flush: 300 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.629414045Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.674712424Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.675444088Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.675484592Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.675498913Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.675513085Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838} {"msg":"Crucible 5ab51020-bab1-4cc9-bbfa-3dc81e9356e2 has session id: c4c760dc-1388-4e65-9674-df618bfb436b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676086483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Upstairs opts: Upstairs UUID: 5ab51020-bab1-4cc9-bbfa-3dc81e9356e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676125066Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Crucible stats registered with UUID: 5ab51020-bab1-4cc9-bbfa-3dc81e9356e2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676149279Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:54.676169711Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676610406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676633728Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838} {"msg":"5ab51020-bab1-4cc9-bbfa-3dc81e9356e2 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676714316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676732408Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"0","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676763421Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"0","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676789713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"1","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676816746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"1","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676840739Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"2","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676863031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"2","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676909556Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.67694975Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.676981363Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50105)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.677004885Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47221)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.677027757Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45066)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.677236119Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.67784267Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"2","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.677874393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"0","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.678708327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"0","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.67873985Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"2","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.679310218Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.679410918Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"1","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.67943035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.679961073Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"1","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[0]R flush_numbers[0..12]: [284, 282, 292, 298, 295, 300, 291, 277, 294, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680490867Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[0]R generation[0..12]: [156, 155, 158, 160, 160, 161, 158, 153, 160, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680509399Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.68052661Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[1]R flush_numbers[0..12]: [284, 282, 292, 298, 295, 300, 291, 277, 294, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680540302Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[1]R generation[0..12]: [156, 155, 158, 160, 160, 161, 158, 153, 160, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680553733Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680567405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[2]R flush_numbers[0..12]: [284, 282, 292, 298, 295, 300, 291, 277, 294, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680580846Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[2]R generation[0..12]: [156, 155, 158, 160, 160, 161, 158, 153, 160, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680594407Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680608169Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:38:54.68062082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680639832Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680652223Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680664794Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"":"downstairs","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680677696Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680689447Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"0","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680702618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"1","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.68071762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"client":"2","":"downstairs","session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680730621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680742332Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"5ab51020-bab1-4cc9-bbfa-3dc81e9356e2 is now active with session: c4c760dc-1388-4e65-9674-df618bfb436b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680754733Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680766325Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838,"session_id":"c4c760dc-1388-4e65-9674-df618bfb436b"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:54.680778056Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 439 len 4096 data: 12 02/30 Write block 75 len 16384 data: 15 18 12 12 03/30 Read block 550 len 40960 04/30 Read block 158 len 32768 05/30 Read block 124 len 24576 06/30 Write block 118 len 36864 data: 15 14 20 20 17 16 16 14 10 07/30 Read block 450 len 40960 08/30 Write block 482 len 32768 data: 10 7 8 10 9 11 14 12 09/30 Read block 287 len 28672 10/30 Write block 46 len 4096 data: 12 11/30 Read block 559 len 32768 12/30 Read block 389 len 8192 13/30 Read block 509 len 20480 14/30 Read block 356 len 20480 15/30 Read block 246 len 12288 16/30 Write block 513 len 4096 data: 12 17/30 Read block 406 len 32768 18/30 Write block 65 len 28672 data: 11 11 12 13 13 14 15 19/30 Flush 20/30 Write block 510 len 36864 data: 12 9 11 13 15 14 13 11 12 21/30 Flush 22/30 Write block 116 len 28672 data: 12 14 16 15 21 21 18 23/30 Write block 154 len 20480 data: 14 13 13 14 14 24/30 Read block 539 len 28672 25/30 Read block 386 len 24576 26/30 Read block 395 len 12288 27/30 Read block 391 len 20480 28/30 Read block 136 len 32768 29/30 Read block 190 len 8192 30/30 Write block 52 len 24576 data: 13 11 15 15 14 14 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:54.702126289Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 162 162 162 301 301 301 T T T 3 060-079 162 162 162 301 301 301 F F F 4 080-099 160 160 160 295 295 295 F F F 5 100-119 162 162 162 301 301 301 T T T 6 120-139 162 162 162 301 301 301 T T T 7 140-159 153 153 153 277 277 277 T T T 8 160-179 160 160 160 294 294 294 F F F 9 180-199 158 158 158 287 287 287 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 158 158 158 291 291 291 F F F 12 240-259 158 158 158 290 290 290 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 159 159 159 293 293 293 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 156 156 156 283 283 283 F F F 17 340-359 161 161 161 300 300 300 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 159 159 159 293 293 293 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 162 162 162 301 301 301 F F F 22 440-459 161 161 161 300 300 300 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 162 162 162 301 301 301 F F F 25 500-519 162 162 162 302 302 302 F F F 26 520-539 160 160 160 299 299 299 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 162, Max flush: 302 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 163 163 163 303 303 303 F F F 3 060-079 162 162 162 301 301 301 F F F 4 080-099 160 160 160 295 295 295 F F F 5 100-119 163 163 163 303 303 303 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 160 160 160 294 294 294 F F F 9 180-199 158 158 158 287 287 287 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 158 158 158 291 291 291 F F F 12 240-259 158 158 158 290 290 290 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 159 159 159 293 293 293 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 156 156 156 283 283 283 F F F 17 340-359 161 161 161 300 300 300 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 159 159 159 293 293 293 F F F 20 400-419 154 154 154 281 281 281 F F F 21 420-439 162 162 162 301 301 301 F F F 22 440-459 161 161 161 300 300 300 F F F 23 460-479 157 157 157 286 286 286 F F F 24 480-499 162 162 162 301 301 301 F F F 25 500-519 162 162 162 302 302 302 F F F 26 520-539 160 160 160 299 299 299 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 163, Max flush: 303 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:58.94239773Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:58.985617189Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:58.986051812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.017619426Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.018402485Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.01844484Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.018469852Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.018494835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846} {"msg":"Crucible b36415bd-ac8a-41c6-8b39-f0ebefddd99c has session id: efadef50-51f6-4659-b0bb-b04c273b30d2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019133099Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Upstairs opts: Upstairs UUID: b36415bd-ac8a-41c6-8b39-f0ebefddd99c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019170623Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Crucible stats registered with UUID: b36415bd-ac8a-41c6-8b39-f0ebefddd99c","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019187865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:59.019200296Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.01963983Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019672023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846} {"msg":"b36415bd-ac8a-41c6-8b39-f0ebefddd99c active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019744041Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019769513Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"0","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019799486Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"0","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019814628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"1","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019828819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"1","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.0198418Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"2","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019855852Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"2","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019868863Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019882315Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.019905017Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40171)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.020164483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41675)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.020211288Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:52394)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.02023725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.021015269Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"1","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.021054713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"0","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.021089576Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"2","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.021743752Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"2","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.021773895Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"0","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.021789897Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"1","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022387907Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.02241066Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[0]R flush_numbers[0..12]: [284, 282, 303, 301, 295, 303, 303, 303, 294, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022470076Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[0]R generation[0..12]: [156, 155, 163, 162, 160, 163, 163, 163, 160, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022490138Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022504449Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[1]R flush_numbers[0..12]: [284, 282, 303, 301, 295, 303, 303, 303, 294, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022518521Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[1]R generation[0..12]: [156, 155, 163, 162, 160, 163, 163, 163, 160, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022541933Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022568315Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[2]R flush_numbers[0..12]: [284, 282, 303, 301, 295, 303, 303, 303, 294, 287, 291, 291]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022592038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[2]R generation[0..12]: [156, 155, 163, 162, 160, 163, 163, 163, 160, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022606779Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:38:59.022630732Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022662195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022687367Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022701509Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.02271418Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"":"downstairs","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022733122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022754234Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"0","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022778217Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"1","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022800599Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"client":"2","":"downstairs","session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.02281469Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022831892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"b36415bd-ac8a-41c6-8b39-f0ebefddd99c is now active with session: efadef50-51f6-4659-b0bb-b04c273b30d2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022853744Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022875717Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846,"session_id":"efadef50-51f6-4659-b0bb-b04c273b30d2"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:38:59.022896729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 82 len 12288 data: 11 12 12 02/30 Read block 202 len 8192 03/30 Read block 78 len 16384 04/30 Write block 336 len 40960 data: 15 11 10 8 8 7 9 10 6 6 05/30 Write block 391 len 24576 data: 12 13 12 10 11 12 06/30 Read block 81 len 40960 07/30 Read block 332 len 20480 08/30 Flush 09/30 Write block 466 len 40960 data: 15 15 16 15 15 16 12 9 8 8 10/30 Read block 306 len 32768 11/30 Write block 426 len 16384 data: 16 13 12 13 12/30 Read block 24 len 40960 13/30 Read block 517 len 24576 14/30 Write block 494 len 12288 data: 10 9 9 15/30 Read block 499 len 40960 16/30 Read block 500 len 4096 17/30 Read block 172 len 16384 18/30 Write block 484 len 8192 data: 9 11 19/30 Read block 454 len 16384 20/30 Read block 251 len 36864 21/30 Write block 235 len 36864 data: 17 18 16 16 15 11 10 7 6 22/30 Read block 345 len 40960 23/30 Read block 333 len 12288 24/30 Write block 412 len 16384 data: 15 15 14 14 25/30 Read block 314 len 32768 26/30 Read block 121 len 36864 27/30 Read block 90 len 32768 28/30 Write block 49 len 20480 data: 11 12 14 14 12 29/30 Read block 21 len 20480 30/30 Write block 456 len 32768 data: 12 13 11 9 7 10 10 8 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 10 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:38:59.045107319Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 163 163 163 303 303 303 T T T 3 060-079 162 162 162 301 301 301 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 163 163 163 303 303 303 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 160 160 160 294 294 294 F F F 9 180-199 158 158 158 287 287 287 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 158 158 158 291 291 291 T T T 12 240-259 158 158 158 290 290 290 T T T 13 260-279 158 158 158 290 290 290 F F F 14 280-299 159 159 159 293 293 293 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 164 164 164 304 304 304 F F F 17 340-359 164 164 164 304 304 304 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 164 164 164 304 304 304 F F F 20 400-419 154 154 154 281 281 281 T T T 21 420-439 162 162 162 301 301 301 T T T 22 440-459 161 161 161 300 300 300 T T T 23 460-479 157 157 157 286 286 286 T T T 24 480-499 162 162 162 301 301 301 T T T 25 500-519 162 162 162 302 302 302 F F F 26 520-539 160 160 160 299 299 299 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 164, Max flush: 304 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 165 165 165 305 305 305 F F F 3 060-079 162 162 162 301 301 301 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 163 163 163 303 303 303 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 160 160 160 294 294 294 F F F 9 180-199 158 158 158 287 287 287 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 165 165 165 305 305 305 F F F 12 240-259 165 165 165 305 305 305 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 159 159 159 293 293 293 F F F 15 300-319 154 154 154 281 281 281 F F F 16 320-339 164 164 164 304 304 304 F F F 17 340-359 164 164 164 304 304 304 F F F 18 360-379 147 147 147 263 263 263 F F F 19 380-399 164 164 164 304 304 304 F F F 20 400-419 165 165 165 305 305 305 F F F 21 420-439 165 165 165 305 305 305 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 165 165 165 305 305 305 F F F 25 500-519 162 162 162 302 302 302 F F F 26 520-539 160 160 160 299 299 299 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 165, Max flush: 305 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.335200995Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.380018915Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.380812815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.38085673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.380872401Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.380886893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854} {"msg":"Crucible 4af4f141-4451-490b-9419-01df63ed8b4d has session id: bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.381440429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Upstairs opts: Upstairs UUID: 4af4f141-4451-490b-9419-01df63ed8b4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.381483693Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Crucible stats registered with UUID: 4af4f141-4451-490b-9419-01df63ed8b4d","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.381506165Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:03.381526817Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382084303Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382110316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854} {"msg":"4af4f141-4451-490b-9419-01df63ed8b4d active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382184624Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382202215Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"0","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382228298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"0","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382252931Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"1","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382279743Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"1","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382362022Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"2","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382394115Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"2","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382422178Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382455281Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382481454Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"ds_connection connected from Ok(127.0.0.1:65122)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382509096Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"ds_connection connected from Ok(127.0.0.1:35615)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382535679Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"ds_connection connected from Ok(127.0.0.1:36934)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.382563012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.383065233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"1","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.383195035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"0","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.383685105Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"1","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.383805937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"0","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.384274154Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.384322399Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.384832791Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"2","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385337242Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"2","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[0]R flush_numbers[0..12]: [284, 282, 305, 301, 304, 303, 303, 303, 294, 287, 291, 305]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385849473Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[0]R generation[0..12]: [156, 155, 165, 162, 164, 163, 163, 163, 160, 158, 158, 165]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385872606Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385890718Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[1]R flush_numbers[0..12]: [284, 282, 305, 301, 304, 303, 303, 303, 294, 287, 291, 305]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385905119Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[1]R generation[0..12]: [156, 155, 165, 162, 164, 163, 163, 163, 160, 158, 158, 165]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.38591897Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385933102Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[2]R flush_numbers[0..12]: [284, 282, 305, 301, 304, 303, 303, 303, 294, 287, 291, 305]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385946533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[2]R generation[0..12]: [156, 155, 165, 162, 164, 163, 163, 163, 160, 158, 158, 165]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385959784Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:39:03.385973376Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.385992118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386004869Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.38601741Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386030092Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"":"downstairs","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386042893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386054454Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"0","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386067325Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"1","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-02-27T17:39:03.386081877Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"client":"2","":"downstairs","session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386096998Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386109009Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"4af4f141-4451-490b-9419-01df63ed8b4d is now active with session: bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386121471Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386133112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854,"session_id":"bcbc57ed-e75f-4dc3-a6cb-4e1545a39e49"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:03.386144863Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 335 len 8192 data: 14 16 02/30 Write block 504 len 40960 data: 10 12 11 7 7 12 13 10 12 14 03/30 Write block 379 len 4096 data: 10 04/30 Write block 168 len 40960 data: 12 12 11 10 10 10 12 13 11 7 05/30 Write block 487 len 24576 data: 12 15 13 10 11 10 06/30 Write block 496 len 16384 data: 10 9 8 9 07/30 Read block 426 len 24576 08/30 Write block 297 len 4096 data: 12 09/30 Write block 312 len 28672 data: 14 10 9 12 12 13 14 10/30 Read block 351 len 32768 11/30 Read block 366 len 32768 12/30 Read block 261 len 12288 13/30 Write block 228 len 40960 data: 16 14 15 14 12 14 14 18 19 17 14/30 Read block 384 len 12288 15/30 Write block 10 len 16384 data: 16 13 15 17 16/30 Write block 191 len 12288 data: 9 10 13 17/30 Read block 58 len 32768 18/30 Read block 473 len 40960 19/30 Read block 4 len 4096 20/30 Read block 518 len 12288 21/30 Read block 336 len 40960 22/30 Read block 230 len 12288 23/30 Read block 52 len 40960 24/30 Read block 209 len 36864 25/30 Read block 490 len 4096 26/30 Read block 275 len 40960 27/30 Read block 368 len 4096 28/30 Read block 519 len 20480 29/30 Read block 449 len 4096 30/30 Read block 401 len 12288 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:03.406430529Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 284 284 284 T T T 1 020-039 155 155 155 282 282 282 F F F 2 040-059 165 165 165 305 305 305 F F F 3 060-079 162 162 162 301 301 301 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 163 163 163 303 303 303 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 160 160 160 294 294 294 T T T 9 180-199 158 158 158 287 287 287 T T T 10 200-219 158 158 158 291 291 291 F F F 11 220-239 165 165 165 305 305 305 T T T 12 240-259 165 165 165 305 305 305 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 159 159 159 293 293 293 T T T 15 300-319 154 154 154 281 281 281 T T T 16 320-339 164 164 164 304 304 304 T T T 17 340-359 164 164 164 304 304 304 F F F 18 360-379 147 147 147 263 263 263 T T T 19 380-399 164 164 164 304 304 304 F F F 20 400-419 165 165 165 305 305 305 F F F 21 420-439 165 165 165 305 305 305 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 165 165 165 305 305 305 T T T 25 500-519 162 162 162 302 302 302 T T T 26 520-539 160 160 160 299 299 299 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 165, Max flush: 305 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 306 306 306 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 165 165 165 305 305 305 F F F 3 060-079 162 162 162 301 301 301 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 163 163 163 303 303 303 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 166 166 166 306 306 306 F F F 9 180-199 166 166 166 306 306 306 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 166 166 166 306 306 306 F F F 12 240-259 165 165 165 305 305 305 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 166 166 166 306 306 306 F F F 15 300-319 166 166 166 306 306 306 F F F 16 320-339 166 166 166 306 306 306 F F F 17 340-359 164 164 164 304 304 304 F F F 18 360-379 166 166 166 306 306 306 F F F 19 380-399 164 164 164 304 304 304 F F F 20 400-419 165 165 165 305 305 305 F F F 21 420-439 165 165 165 305 305 305 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 166 166 166 306 306 306 F F F 25 500-519 166 166 166 306 306 306 F F F 26 520-539 160 160 160 299 299 299 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 166, Max flush: 306 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.678465564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:07.727527382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.727994269Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.759836141Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.760602758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.760645462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.760662814Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.760680556Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862} {"msg":"Crucible 12412088-6922-474c-b49b-913560949aaf has session id: b857001c-5c2f-49fd-ab3a-000673c09202","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.761385097Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Upstairs opts: Upstairs UUID: 12412088-6922-474c-b49b-913560949aaf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.761425871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Crucible stats registered with UUID: 12412088-6922-474c-b49b-913560949aaf","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.761454094Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:07.761474886Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.761955104Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.761981707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862} {"msg":"12412088-6922-474c-b49b-913560949aaf active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762054404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762072526Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"0","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762095598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"0","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.7621095Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"1","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762123651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"1","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762136713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"2","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762150884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"2","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762164275Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762178067Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762198539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"ds_connection connected from Ok(127.0.0.1:52821)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762381617Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"ds_connection connected from Ok(127.0.0.1:63287)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762560555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"ds_connection connected from Ok(127.0.0.1:37487)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.762599429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.763194699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"1","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.763232303Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"2","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.763259356Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"0","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.76389761Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"2","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.763933364Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"0","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.763961606Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"1","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764559687Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.76458653Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[0]R flush_numbers[0..12]: [306, 282, 305, 301, 304, 303, 303, 303, 306, 306, 291, 306]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764629644Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[0]R generation[0..12]: [166, 155, 165, 162, 164, 163, 163, 163, 166, 166, 158, 166]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764647386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764661797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[1]R flush_numbers[0..12]: [306, 282, 305, 301, 304, 303, 303, 303, 306, 306, 291, 306]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764675069Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[1]R generation[0..12]: [166, 155, 165, 162, 164, 163, 163, 163, 166, 166, 158, 166]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.76468837Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764705402Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[2]R flush_numbers[0..12]: [306, 282, 305, 301, 304, 303, 303, 303, 306, 306, 291, 306]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764719323Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[2]R generation[0..12]: [166, 155, 165, 162, 164, 163, 163, 163, 166, 166, 158, 166]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764732744Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764746226Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Max found gen is 167","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764758607Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Generation requested: 168 >= found:167","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764771568Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764784449Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764797151Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"":"downstairs","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"Wait for a query_work_queue command to finish before sending IO msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764819103Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764831054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"0","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764844336Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"1","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764857317Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"client":"2","":"downstairs","session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764870378Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.76488231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"12412088-6922-474c-b49b-913560949aaf is now active with session: b857001c-5c2f-49fd-ab3a-000673c09202","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764894781Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764908472Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862,"session_id":"b857001c-5c2f-49fd-ab3a-000673c09202"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:07.764920333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 329 len 24576 data: 9 9 9 14 11 15 02/30 Read block 469 len 4096 03/30 Write block 174 len 36864 data: 13 14 12 8 6 9 9 10 9 04/30 Write block 379 len 24576 data: 11 9 10 13 16 17 05/30 Read block 331 len 8192 06/30 Read block 33 len 4096 07/30 Flush 08/30 Write block 531 len 24576 data: 16 13 11 9 9 9 09/30 Read block 543 len 40960 10/30 Read block 258 len 8192 11/30 Read block 519 len 16384 12/30 Read block 194 len 8192 13/30 Read block 280 len 36864 14/30 Read block 341 len 32768 15/30 Read block 457 len 16384 16/30 Write block 104 len 16384 data: 10 12 12 11 17/30 Flush 18/30 Write block 192 len 8192 data: 11 14 19/30 Flush 20/30 Write block 330 len 36864 data: 10 10 15 12 16 15 17 12 11 21/30 Read block 102 len 32768 22/30 Read block 163 len 24576 23/30 Read block 447 len 28672 24/30 Read block 92 len 32768 25/30 Flush 26/30 Write block 189 len 20480 data: 7 6 10 12 15 27/30 Read block 236 len 28672 28/30 Read block 158 len 40960 29/30 Write block 438 len 4096 data: 10 30/30 Read block 82 len 24576 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 5 5 3 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:07.785978177Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 306 306 306 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 165 165 165 305 305 305 F F F 3 060-079 162 162 162 301 301 301 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 168 168 168 308 308 308 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 168 168 168 307 307 307 F F F 9 180-199 168 168 168 309 309 309 T T T 10 200-219 158 158 158 291 291 291 F F F 11 220-239 166 166 166 306 306 306 F F F 12 240-259 165 165 165 305 305 305 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 166 166 166 306 306 306 F F F 15 300-319 166 166 166 306 306 306 F F F 16 320-339 168 168 168 310 310 310 F F F 17 340-359 164 164 164 304 304 304 F F F 18 360-379 168 168 168 307 307 307 F F F 19 380-399 168 168 168 307 307 307 F F F 20 400-419 165 165 165 305 305 305 F F F 21 420-439 165 165 165 305 305 305 T T T 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 166 166 166 306 306 306 F F F 25 500-519 166 166 166 306 306 306 F F F 26 520-539 168 168 168 308 308 308 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 168, Max flush: 310 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 306 306 306 F F F 1 020-039 155 155 155 282 282 282 F F F 2 040-059 165 165 165 305 305 305 F F F 3 060-079 162 162 162 301 301 301 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 168 168 168 308 308 308 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 168 168 168 307 307 307 F F F 9 180-199 169 169 169 311 311 311 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 166 166 166 306 306 306 F F F 12 240-259 165 165 165 305 305 305 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 166 166 166 306 306 306 F F F 15 300-319 166 166 166 306 306 306 F F F 16 320-339 168 168 168 310 310 310 F F F 17 340-359 164 164 164 304 304 304 F F F 18 360-379 168 168 168 307 307 307 F F F 19 380-399 168 168 168 307 307 307 F F F 20 400-419 165 165 165 305 305 305 F F F 21 420-439 169 169 169 311 311 311 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 166 166 166 306 306 306 F F F 25 500-519 166 166 166 306 306 306 F F F 26 520-539 168 168 168 308 308 308 F F F 27 540-559 161 161 161 300 300 300 F F F 28 560-579 161 161 161 300 300 300 F F F 29 580-599 152 152 152 273 273 273 F F F Max gen: 169, Max flush: 311 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.049035896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.094246186Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.09498192Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.095024884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.095039546Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.095053777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870} {"msg":"Crucible 5c91a3c1-f56e-4d88-9169-d8874190b4ff has session id: 52b5428f-9b54-422b-8ba7-149bb2476fc3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.095705023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Upstairs opts: Upstairs UUID: 5c91a3c1-f56e-4d88-9169-d8874190b4ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.095746917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Crucible stats registered with UUID: 5c91a3c1-f56e-4d88-9169-d8874190b4ff","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.09577096Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:12.095791312Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096216555Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096243247Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870} {"msg":"5c91a3c1-f56e-4d88-9169-d8874190b4ff active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096312174Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096329456Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"0","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096359729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"0","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096387242Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"1","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096415115Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"1","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096441427Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"2","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.09646809Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"2","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096493963Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096523626Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096548198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48769)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096589662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:41443)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.09666815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"ds_connection connected from Ok(127.0.0.1:59956)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.096702964Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.097205774Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"1","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.097242268Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"0","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.097823397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"1","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.097975402Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"0","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.098393484Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.09855211Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.098699125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"2","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099214057Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"2","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[0]R flush_numbers[0..12]: [306, 282, 305, 301, 304, 308, 303, 303, 307, 311, 291, 306]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.09973995Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[0]R generation[0..12]: [166, 155, 165, 162, 164, 168, 163, 163, 168, 169, 158, 166]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099766693Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099781934Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[1]R flush_numbers[0..12]: [306, 282, 305, 301, 304, 308, 303, 303, 307, 311, 291, 306]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099795766Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[1]R generation[0..12]: [166, 155, 165, 162, 164, 168, 163, 163, 168, 169, 158, 166]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099809747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099823478Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[2]R flush_numbers[0..12]: [306, 282, 305, 301, 304, 308, 303, 303, 307, 311, 291, 306]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.09983693Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[2]R generation[0..12]: [166, 155, 165, 162, 164, 168, 163, 163, 168, 169, 158, 166]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099850681Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099871193Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099883934Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099896646Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099909187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099921678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"":"downstairs","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099934359Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099946041Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"0","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099958692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099979804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"client":"2","":"downstairs","session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.099992825Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.100004807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"5c91a3c1-f56e-4d88-9169-d8874190b4ff is now active with session: 52b5428f-9b54-422b-8ba7-149bb2476fc3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.100017638Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.100029399Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870,"session_id":"52b5428f-9b54-422b-8ba7-149bb2476fc3"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:12.10004126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 580 len 32768 data: 7 8 10 10 11 9 9 9 02/30 Write block 538 len 12288 data: 14 13 13 03/30 Write block 196 len 4096 data: 13 04/30 Write block 422 len 24576 data: 15 13 10 13 17 14 05/30 Write block 66 len 8192 data: 12 13 06/30 Read block 123 len 28672 07/30 Flush 08/30 Write block 78 len 8192 data: 13 14 09/30 Write block 1 len 4096 data: 5 10/30 Read block 235 len 32768 11/30 Write block 299 len 8192 data: 14 17 12/30 Write block 32 len 36864 data: 13 13 12 12 12 13 12 13 11 13/30 Write block 570 len 28672 data: 13 12 16 16 14 15 12 14/30 Read block 298 len 40960 15/30 Read block 285 len 8192 16/30 Read block 128 len 32768 17/30 Write block 481 len 12288 data: 7 11 8 18/30 Read block 205 len 12288 19/30 Write block 228 len 12288 data: 17 15 16 20/30 Write block 36 len 12288 data: 13 14 13 21/30 Read block 17 len 12288 22/30 Write block 504 len 28672 data: 11 13 12 8 8 13 14 23/30 Write block 61 len 28672 data: 13 12 13 13 12 13 14 24/30 Read block 497 len 20480 25/30 Read block 587 len 40960 26/30 Read block 544 len 24576 27/30 Read block 486 len 20480 28/30 Write block 320 len 40960 data: 16 20 18 19 18 16 15 13 8 10 29/30 Read block 590 len 40960 30/30 Read block 222 len 40960 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 23 22 23 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:12.11917398Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 306 306 306 T T T 1 020-039 155 155 155 282 282 282 T T T 2 040-059 165 165 165 305 305 305 T T T 3 060-079 170 170 170 312 312 312 T T T 4 080-099 164 164 164 304 304 304 F F F 5 100-119 168 168 168 308 308 308 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 168 168 168 307 307 307 F F F 9 180-199 170 170 170 312 312 312 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 166 166 166 306 306 306 T T T 12 240-259 165 165 165 305 305 305 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 166 166 166 306 306 306 T T T 15 300-319 166 166 166 306 306 306 T T T 16 320-339 168 168 168 310 310 310 T T T 17 340-359 164 164 164 304 304 304 F F F 18 360-379 168 168 168 307 307 307 F F F 19 380-399 168 168 168 307 307 307 F F F 20 400-419 165 165 165 305 305 305 F F F 21 420-439 170 170 170 312 312 312 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 166 166 166 306 306 306 T T T 25 500-519 166 166 166 306 306 306 T T T 26 520-539 170 170 170 312 312 312 F F F 27 540-559 170 170 170 312 312 312 F F F 28 560-579 161 161 161 300 300 300 T T T 29 580-599 170 170 170 312 312 312 F F F Max gen: 170, Max flush: 312 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 171 171 171 313 313 313 F F F 2 040-059 171 171 171 313 313 313 F F F 3 060-079 171 171 171 313 313 313 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 168 168 168 308 308 308 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 168 168 168 307 307 307 F F F 9 180-199 170 170 170 312 312 312 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 171 171 171 313 313 313 F F F 12 240-259 165 165 165 305 305 305 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 171 171 171 313 313 313 F F F 15 300-319 171 171 171 313 313 313 F F F 16 320-339 171 171 171 313 313 313 F F F 17 340-359 164 164 164 304 304 304 F F F 18 360-379 168 168 168 307 307 307 F F F 19 380-399 168 168 168 307 307 307 F F F 20 400-419 165 165 165 305 305 305 F F F 21 420-439 170 170 170 312 312 312 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 170 170 170 312 312 312 F F F 27 540-559 170 170 170 312 312 312 F F F 28 560-579 171 171 171 313 313 313 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 171, Max flush: 313 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.392956021Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:26.436952728Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.437484392Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.469601761Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.470266098Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.470301902Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.470318023Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.470334865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878} {"msg":"Crucible addea91d-997c-4173-be59-775c0f2f322e has session id: 128fa49c-4834-4963-877b-566904c8ebc6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.470919824Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Upstairs opts: Upstairs UUID: addea91d-997c-4173-be59-775c0f2f322e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.470964649Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Crucible stats registered with UUID: addea91d-997c-4173-be59-775c0f2f322e","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.470988931Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:26.471030185Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471466439Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471491662Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878} {"msg":"addea91d-997c-4173-be59-775c0f2f322e active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.47157771Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471595342Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"0","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471621715Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"0","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471635466Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"1","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471659649Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"1","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471684381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"2","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471712044Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"2","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471741487Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.47176947Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471797213Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46630)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471825155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42918)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.471848538Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"ds_connection connected from Ok(127.0.0.1:47582)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.47187094Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.472531877Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"1","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.47256613Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"2","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.472820546Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"0","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.473212645Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"1","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.473424357Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"2","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.473626037Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"0","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.473798414Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.473929988Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[0]R flush_numbers[0..12]: [313, 313, 313, 313, 304, 308, 303, 303, 307, 312, 291, 313]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474244289Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[0]R generation[0..12]: [171, 171, 171, 171, 164, 168, 163, 163, 168, 170, 158, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474274652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474290924Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[1]R flush_numbers[0..12]: [313, 313, 313, 313, 304, 308, 303, 303, 307, 312, 291, 313]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474304806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[1]R generation[0..12]: [171, 171, 171, 171, 164, 168, 163, 163, 168, 170, 158, 171]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474318507Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474337679Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[2]R flush_numbers[0..12]: [313, 313, 313, 313, 304, 308, 303, 303, 307, 312, 291, 313]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.47435221Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[2]R generation[0..12]: [171, 171, 171, 171, 164, 168, 163, 163, 168, 170, 158, 171]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:39:26.474366052Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474385594Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474398335Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474411696Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474424527Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474437099Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"":"downstairs","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.47444994Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474461671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"0","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474474992Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"1","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474487894Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"client":"2","":"downstairs","session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474500765Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474512616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"addea91d-997c-4173-be59-775c0f2f322e is now active with session: 128fa49c-4834-4963-877b-566904c8ebc6","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474525048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.474536739Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878,"session_id":"128fa49c-4834-4963-877b-566904c8ebc6"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:26.47455113Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 246 len 16384 data: 14 13 13 17 02/30 Flush 03/30 Write block 434 len 4096 data: 9 04/30 Read block 432 len 40960 05/30 Write block 244 len 40960 data: 8 11 15 14 14 18 19 19 20 23 06/30 Read block 507 len 28672 07/30 Read block 268 len 12288 08/30 Read block 69 len 12288 09/30 Write block 21 len 32768 data: 10 11 10 8 10 10 14 12 10/30 Read block 81 len 12288 11/30 Read block 330 len 40960 12/30 Read block 268 len 40960 13/30 Read block 279 len 16384 14/30 Read block 486 len 24576 15/30 Read block 26 len 12288 16/30 Write block 187 len 36864 data: 10 9 8 7 11 13 16 15 17 17/30 Write block 407 len 8192 data: 9 13 18/30 Write block 74 len 4096 data: 14 19/30 Read block 294 len 36864 20/30 Write block 334 len 12288 data: 17 16 18 21/30 Flush 22/30 Write block 335 len 36864 data: 17 19 13 12 9 9 8 10 11 23/30 Write block 244 len 28672 data: 9 12 16 15 15 19 20 24/30 Read block 176 len 32768 25/30 Read block 546 len 12288 26/30 Write block 104 len 36864 data: 11 13 13 12 13 16 14 13 13 27/30 Read block 245 len 16384 28/30 Read block 348 len 8192 29/30 Write block 233 len 32768 data: 15 15 19 20 18 17 16 12 30/30 Write block 547 len 24576 data: 15 21 23 22 18 16 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:26.496079882Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 172 172 172 315 315 315 F F F 2 040-059 171 171 171 313 313 313 F F F 3 060-079 172 172 172 315 315 315 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 168 168 168 308 308 308 T T T 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 168 168 168 307 307 307 F F F 9 180-199 172 172 172 315 315 315 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 171 171 171 313 313 313 T T T 12 240-259 172 172 172 315 315 315 T T T 13 260-279 158 158 158 290 290 290 F F F 14 280-299 171 171 171 313 313 313 F F F 15 300-319 171 171 171 313 313 313 F F F 16 320-339 172 172 172 315 315 315 T T T 17 340-359 164 164 164 304 304 304 T T T 18 360-379 168 168 168 307 307 307 F F F 19 380-399 168 168 168 307 307 307 F F F 20 400-419 172 172 172 315 315 315 F F F 21 420-439 172 172 172 315 315 315 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 170 170 170 312 312 312 F F F 27 540-559 170 170 170 312 312 312 T T T 28 560-579 171 171 171 313 313 313 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 172, Max flush: 315 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 172 172 172 315 315 315 F F F 2 040-059 171 171 171 313 313 313 F F F 3 060-079 172 172 172 315 315 315 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 173 173 173 316 316 316 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 168 168 168 307 307 307 F F F 9 180-199 172 172 172 315 315 315 F F F 10 200-219 158 158 158 291 291 291 F F F 11 220-239 173 173 173 316 316 316 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 158 158 158 290 290 290 F F F 14 280-299 171 171 171 313 313 313 F F F 15 300-319 171 171 171 313 313 313 F F F 16 320-339 173 173 173 316 316 316 F F F 17 340-359 173 173 173 316 316 316 F F F 18 360-379 168 168 168 307 307 307 F F F 19 380-399 168 168 168 307 307 307 F F F 20 400-419 172 172 172 315 315 315 F F F 21 420-439 172 172 172 315 315 315 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 170 170 170 312 312 312 F F F 27 540-559 173 173 173 316 316 316 F F F 28 560-579 171 171 171 313 313 313 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 173, Max flush: 316 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.775133884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.820989879Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.82179243Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.821834214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.821852586Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.821867277Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886} {"msg":"Crucible 9fed38e8-63d9-44bc-b47e-cb3c620f8c5e has session id: 7d478162-4459-4460-85c0-6d2b0ac12967","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.822428514Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Upstairs opts: Upstairs UUID: 9fed38e8-63d9-44bc-b47e-cb3c620f8c5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.822467588Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Crucible stats registered with UUID: 9fed38e8-63d9-44bc-b47e-cb3c620f8c5e","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.82248978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:30.822511672Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.822966298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.822993221Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886} {"msg":"9fed38e8-63d9-44bc-b47e-cb3c620f8c5e active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823049077Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823072149Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"0","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823095391Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"0","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823109353Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"1","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823123404Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"1","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823136685Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"2","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823150917Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"2","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823166819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.82318035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823202582Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"ds_connection connected from Ok(127.0.0.1:52202)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823305382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"ds_connection connected from Ok(127.0.0.1:47899)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823339206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"ds_connection connected from Ok(127.0.0.1:54617)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.823367099Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.824052928Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"1","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.824080071Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"0","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.824745488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"1","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.82477163Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"0","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.825306064Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"2","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.825477432Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.825495333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.825853399Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"2","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[0]R flush_numbers[0..12]: [313, 315, 313, 315, 304, 316, 303, 303, 307, 315, 291, 316]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826376312Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[0]R generation[0..12]: [171, 172, 171, 172, 164, 173, 163, 163, 168, 172, 158, 173]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826397604Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826412496Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[1]R flush_numbers[0..12]: [313, 315, 313, 315, 304, 316, 303, 303, 307, 315, 291, 316]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826426667Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[1]R generation[0..12]: [171, 172, 171, 172, 164, 173, 163, 163, 168, 172, 158, 173]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826449839Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826475652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[2]R flush_numbers[0..12]: [313, 315, 313, 315, 304, 316, 303, 303, 307, 315, 291, 316]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826502825Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[2]R generation[0..12]: [171, 172, 171, 172, 164, 173, 163, 163, 168, 172, 158, 173]","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826534738Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826562041Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826582053Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826595534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826608206Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826628117Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"":"downstairs","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.82665101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826673462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"0","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826698735Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"1","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826713006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"client":"2","":"downstairs","session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826733648Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.82675525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"9fed38e8-63d9-44bc-b47e-cb3c620f8c5e is now active with session: 7d478162-4459-4460-85c0-6d2b0ac12967","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826779183Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826798195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886,"session_id":"7d478162-4459-4460-85c0-6d2b0ac12967"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:30.826810806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 70 len 36864 data: 15 16 13 13 15 16 19 13 14 03/30 Read block 174 len 8192 04/30 Read block 343 len 28672 05/30 Read block 265 len 8192 06/30 Read block 158 len 8192 07/30 Read block 463 len 12288 08/30 Read block 255 len 4096 09/30 Read block 372 len 20480 10/30 Write block 216 len 32768 data: 19 17 17 18 17 17 17 19 11/30 Read block 171 len 4096 12/30 Read block 521 len 40960 13/30 Write block 532 len 24576 data: 14 12 10 10 10 10 14/30 Write block 208 len 28672 data: 9 11 14 14 15 16 14 15/30 Write block 222 len 8192 data: 18 20 16/30 Write block 314 len 12288 data: 10 13 13 17/30 Write block 373 len 12288 data: 12 11 9 18/30 Read block 68 len 24576 19/30 Write block 347 len 20480 data: 10 12 11 14 12 20/30 Read block 575 len 20480 21/30 Read block 498 len 24576 22/30 Write block 111 len 16384 data: 14 14 11 9 23/30 Read block 358 len 36864 24/30 Write block 277 len 4096 data: 10 25/30 Write block 373 len 12288 data: 13 12 10 26/30 Write block 165 len 24576 data: 12 11 12 13 13 12 27/30 Read block 140 len 16384 28/30 Read block 361 len 24576 29/30 Write block 203 len 12288 data: 11 10 11 30/30 Read block 363 len 16384 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 3 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 27 29 27 83 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:30.846190711Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 172 172 172 315 315 315 F F F 2 040-059 171 171 171 313 313 313 F F F 3 060-079 172 172 172 315 315 315 T T T 4 080-099 164 164 164 304 304 304 F F F 5 100-119 173 173 173 316 316 316 T T T 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 168 168 168 307 307 307 T T T 9 180-199 172 172 172 315 315 315 F F F 10 200-219 158 158 158 291 291 291 T T T 11 220-239 173 173 173 316 316 316 T T T 12 240-259 173 173 173 316 316 316 F F F 13 260-279 158 158 158 290 290 290 T T T 14 280-299 171 171 171 313 313 313 F F F 15 300-319 171 171 171 313 313 313 T T T 16 320-339 173 173 173 316 316 316 F F F 17 340-359 173 173 173 316 316 316 T T T 18 360-379 168 168 168 307 307 307 T T T 19 380-399 168 168 168 307 307 307 F F F 20 400-419 172 172 172 315 315 315 F F F 21 420-439 172 172 172 315 315 315 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 170 170 170 312 312 312 T T T 27 540-559 173 173 173 316 316 316 F F F 28 560-579 171 171 171 313 313 313 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 173, Max flush: 316 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 172 172 172 315 315 315 F F F 2 040-059 171 171 171 313 313 313 F F F 3 060-079 174 174 174 317 317 317 F F F 4 080-099 164 164 164 304 304 304 F F F 5 100-119 174 174 174 317 317 317 F F F 6 120-139 163 163 163 303 303 303 F F F 7 140-159 163 163 163 303 303 303 F F F 8 160-179 174 174 174 317 317 317 F F F 9 180-199 172 172 172 315 315 315 F F F 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 174 174 174 317 317 317 F F F 14 280-299 171 171 171 313 313 313 F F F 15 300-319 174 174 174 317 317 317 F F F 16 320-339 173 173 173 316 316 316 F F F 17 340-359 174 174 174 317 317 317 F F F 18 360-379 174 174 174 317 317 317 F F F 19 380-399 168 168 168 307 307 307 F F F 20 400-419 172 172 172 315 315 315 F F F 21 420-439 172 172 172 315 315 315 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 174 174 174 317 317 317 F F F 27 540-559 173 173 173 316 316 316 F F F 28 560-579 171 171 171 313 313 313 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 174, Max flush: 317 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.11926015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.163377079Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.164157648Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.164199812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.164215293Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.164229315Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894} {"msg":"Crucible 8692590b-3d44-4004-a85a-798d84bf93eb has session id: b878d0e0-1be8-45ec-aa7d-a943bc850eb1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.16477806Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Upstairs opts: Upstairs UUID: 8692590b-3d44-4004-a85a-798d84bf93eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.164807363Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Crucible stats registered with UUID: 8692590b-3d44-4004-a85a-798d84bf93eb","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.164822835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:35.164839827Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.16527295Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165295622Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894} {"msg":"8692590b-3d44-4004-a85a-798d84bf93eb active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165378921Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165396233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"0","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165418915Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"0","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165432406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"1","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165446408Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"1","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165459479Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165499873Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165523956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165548248Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165572481Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:49073)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165599073Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:52797)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165621755Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:35.165652758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:35.165680551Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:35.165708134Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.165820235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.166307515Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"1","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.166522866Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"0","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.16696175Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"1","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.167094554Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"0","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.167577353Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:35.167667742Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.166125616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35091)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.166270321Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.16754397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168120548Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[0]R flush_numbers[0..12]: [313, 315, 313, 317, 304, 317, 303, 303, 317, 315, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168710237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[0]R generation[0..12]: [171, 172, 171, 174, 164, 174, 163, 163, 174, 172, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168730349Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168746101Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[1]R flush_numbers[0..12]: [313, 315, 313, 317, 304, 317, 303, 303, 317, 315, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168759682Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[1]R generation[0..12]: [171, 172, 171, 174, 164, 174, 163, 163, 174, 172, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168773283Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168787345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[2]R flush_numbers[0..12]: [313, 315, 313, 317, 304, 317, 303, 303, 317, 315, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168801036Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[2]R generation[0..12]: [171, 172, 171, 174, 164, 174, 163, 163, 174, 172, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168814338Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168827869Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Max found gen is 175","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.16884035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Generation requested: 176 >= found:175","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168853091Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Next flush: 318","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168873074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168885835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"":"downstairs","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168898906Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168912918Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"0","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168926029Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"1","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.1689387Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"client":"2","":"downstairs","session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168951481Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168963533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"8692590b-3d44-4004-a85a-798d84bf93eb is now active with session: b878d0e0-1be8-45ec-aa7d-a943bc850eb1","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168975894Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168987325Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894,"session_id":"b878d0e0-1be8-45ec-aa7d-a943bc850eb1"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:45.168999126Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 395 len 24576 data: 12 13 11 8 8 8 02/30 Read block 237 len 24576 03/30 Read block 288 len 4096 04/30 Read block 398 len 12288 05/30 Write block 157 len 4096 data: 15 06/30 Read block 457 len 12288 07/30 Read block 305 len 20480 08/30 Write block 279 len 8192 data: 10 7 09/30 Read block 96 len 28672 10/30 Write block 425 len 40960 data: 14 18 15 13 14 14 16 12 9 10 11/30 Write block 270 len 32768 data: 9 8 9 9 10 9 10 11 12/30 Read block 92 len 4096 13/30 Write block 129 len 36864 data: 6 7 7 10 12 14 12 13 13 14/30 Read block 469 len 36864 15/30 Write block 280 len 8192 data: 8 6 16/30 Read block 579 len 32768 17/30 Write block 181 len 36864 data: 11 10 7 12 13 13 11 10 9 18/30 Read block 157 len 40960 19/30 Read block 248 len 8192 20/30 Write block 67 len 28672 data: 15 14 14 16 17 14 14 21/30 Flush 22/30 Write block 80 len 16384 data: 15 11 12 13 23/30 Write block 69 len 36864 data: 15 17 18 15 15 16 17 20 14 24/30 Write block 550 len 24576 data: 23 19 17 13 10 12 25/30 Write block 366 len 32768 data: 11 12 13 14 13 16 15 14 26/30 Write block 279 len 12288 data: 11 9 7 27/30 Read block 354 len 8192 28/30 Read block 78 len 16384 29/30 Read block 497 len 24576 30/30 Read block 391 len 32768 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 9 8 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:45.189669941Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 172 172 172 315 315 315 F F F 2 040-059 171 171 171 313 313 313 F F F 3 060-079 176 176 176 318 318 318 T T T 4 080-099 164 164 164 304 304 304 T T T 5 100-119 174 174 174 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 174 174 174 317 317 317 F F F 9 180-199 176 176 176 318 318 318 F F F 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 176 176 176 318 318 318 T T T 14 280-299 176 176 176 318 318 318 T T T 15 300-319 174 174 174 317 317 317 F F F 16 320-339 173 173 173 316 316 316 F F F 17 340-359 174 174 174 317 317 317 F F F 18 360-379 174 174 174 317 317 317 T T T 19 380-399 176 176 176 318 318 318 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 176 176 176 318 318 318 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 174 174 174 317 317 317 F F F 27 540-559 173 173 173 316 316 316 T T T 28 560-579 171 171 171 313 313 313 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 176, Max flush: 318 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 172 172 172 315 315 315 F F F 2 040-059 171 171 171 313 313 313 F F F 3 060-079 177 177 177 319 319 319 F F F 4 080-099 177 177 177 319 319 319 F F F 5 100-119 174 174 174 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 174 174 174 317 317 317 F F F 9 180-199 176 176 176 318 318 318 F F F 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 177 177 177 319 319 319 F F F 14 280-299 177 177 177 319 319 319 F F F 15 300-319 174 174 174 317 317 317 F F F 16 320-339 173 173 173 316 316 316 F F F 17 340-359 174 174 174 317 317 317 F F F 18 360-379 177 177 177 319 319 319 F F F 19 380-399 176 176 176 318 318 318 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 176 176 176 318 318 318 F F F 22 440-459 165 165 165 305 305 305 F F F 23 460-479 165 165 165 305 305 305 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 174 174 174 317 317 317 F F F 27 540-559 177 177 177 319 319 319 F F F 28 560-579 171 171 171 313 313 313 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 177, Max flush: 319 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.441843212Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.486059692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.486818858Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.486863263Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.486877864Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.486891956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902} {"msg":"Crucible 0944164a-3f0d-48eb-83dc-a007184a31df has session id: 31e7bf59-7f9e-44c5-abb7-af75aa9e58d0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.487420729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Upstairs opts: Upstairs UUID: 0944164a-3f0d-48eb-83dc-a007184a31df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.487460003Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Crucible stats registered with UUID: 0944164a-3f0d-48eb-83dc-a007184a31df","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.487482235Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:59.487503298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.48792381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.487956053Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902} {"msg":"0944164a-3f0d-48eb-83dc-a007184a31df active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488042462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488076115Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"0","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488115639Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"0","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488143592Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488170435Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488197048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"2","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.48822542Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"2","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488252053Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488276165Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488299038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54557)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.48832233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34799)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488347133Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:59.488374195Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:59.488401648Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:39:59.488555534Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488677736Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.488992298Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"2","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.489144533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"0","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.489533972Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"2","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.489645864Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"0","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.490063726Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:39:59.490272747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.489372256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40976)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.489498659Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.490524422Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.4913959Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[0]R flush_numbers[0..12]: [313, 315, 313, 319, 319, 317, 318, 318, 317, 318, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492113513Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[0]R generation[0..12]: [171, 172, 171, 177, 177, 174, 176, 176, 174, 176, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492135605Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492151346Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[1]R flush_numbers[0..12]: [313, 315, 313, 319, 319, 317, 318, 318, 317, 318, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492182009Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[1]R generation[0..12]: [171, 172, 171, 177, 177, 174, 176, 176, 174, 176, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492197871Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492211972Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[2]R flush_numbers[0..12]: [313, 315, 313, 319, 319, 317, 318, 318, 317, 318, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492225254Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[2]R generation[0..12]: [171, 172, 171, 177, 177, 174, 176, 176, 174, 176, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492238495Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492252207Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492264918Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"Wait for a query_work_queue command to finish before sending IO msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.49228312Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492296071Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492308622Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"":"downstairs","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492321463Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492333485Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"0","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492354937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"1","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492374899Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"client":"2","":"downstairs","session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.4923887Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492402972Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"0944164a-3f0d-48eb-83dc-a007184a31df is now active with session: 31e7bf59-7f9e-44c5-abb7-af75aa9e58d0","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492415823Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492427694Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902,"session_id":"31e7bf59-7f9e-44c5-abb7-af75aa9e58d0"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:09.492439775Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 564 len 16384 data: 9 12 11 12 02/30 Write block 275 len 4096 data: 10 03/30 Read block 39 len 32768 04/30 Flush 05/30 Write block 563 len 20480 data: 7 10 13 12 13 06/30 Read block 207 len 4096 07/30 Read block 371 len 28672 08/30 Read block 36 len 16384 09/30 Read block 518 len 24576 10/30 Write block 73 len 4096 data: 16 11/30 Read block 88 len 16384 12/30 Read block 137 len 12288 13/30 Read block 133 len 24576 14/30 Read block 26 len 32768 15/30 Write block 175 len 36864 data: 15 13 9 7 10 10 12 11 8 16/30 Flush 17/30 Write block 38 len 32768 data: 14 14 12 13 14 14 11 10 18/30 Read block 297 len 8192 19/30 Flush 20/30 Write block 455 len 24576 data: 11 13 14 12 10 8 21/30 Write block 186 len 40960 data: 14 12 11 10 8 12 14 17 16 18 22/30 Write block 352 len 36864 data: 13 11 11 12 13 15 11 8 12 23/30 Write block 199 len 4096 data: 14 24/30 Read block 76 len 40960 25/30 Read block 165 len 16384 26/30 Write block 540 len 20480 data: 14 17 17 16 18 27/30 Read block 386 len 32768 28/30 Read block 529 len 40960 29/30 Read block 56 len 12288 30/30 Write block 158 len 36864 data: 15 13 10 10 11 14 12 13 12 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:09.513561876Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 178 178 178 322 322 322 F F F 2 040-059 178 178 178 322 322 322 F F F 3 060-079 178 178 178 321 321 321 F F F 4 080-099 177 177 177 319 319 319 F F F 5 100-119 174 174 174 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 T T T 8 160-179 178 178 178 321 321 321 T T T 9 180-199 178 178 178 321 321 321 T T T 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 178 178 178 320 320 320 F F F 14 280-299 177 177 177 319 319 319 F F F 15 300-319 174 174 174 317 317 317 F F F 16 320-339 173 173 173 316 316 316 F F F 17 340-359 174 174 174 317 317 317 T T T 18 360-379 177 177 177 319 319 319 T T T 19 380-399 176 176 176 318 318 318 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 176 176 176 318 318 318 F F F 22 440-459 165 165 165 305 305 305 T T T 23 460-479 165 165 165 305 305 305 T T T 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 174 174 174 317 317 317 F F F 27 540-559 177 177 177 319 319 319 T T T 28 560-579 178 178 178 321 321 321 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 178, Max flush: 322 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 F F F 1 020-039 178 178 178 322 322 322 F F F 2 040-059 178 178 178 322 322 322 F F F 3 060-079 178 178 178 321 321 321 F F F 4 080-099 177 177 177 319 319 319 F F F 5 100-119 174 174 174 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 179 179 179 323 323 323 F F F 8 160-179 179 179 179 323 323 323 F F F 9 180-199 179 179 179 323 323 323 F F F 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 178 178 178 320 320 320 F F F 14 280-299 177 177 177 319 319 319 F F F 15 300-319 174 174 174 317 317 317 F F F 16 320-339 173 173 173 316 316 316 F F F 17 340-359 179 179 179 323 323 323 F F F 18 360-379 179 179 179 323 323 323 F F F 19 380-399 176 176 176 318 318 318 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 176 176 176 318 318 318 F F F 22 440-459 179 179 179 323 323 323 F F F 23 460-479 179 179 179 323 323 323 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 174 174 174 317 317 317 F F F 27 540-559 179 179 179 323 323 323 F F F 28 560-579 178 178 178 321 321 321 F F F 29 580-599 170 170 170 312 312 312 F F F Max gen: 179, Max flush: 323 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.776580961Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:23.820291359Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.820722273Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.853353324Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.854029072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.854071346Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.854088198Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.85410602Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910} {"msg":"Crucible 14ba212f-8a4d-431d-ba70-d7f56c87ec78 has session id: 9819f899-ceb7-4500-be2f-1ace2ee25501","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.854691559Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Upstairs opts: Upstairs UUID: 14ba212f-8a4d-431d-ba70-d7f56c87ec78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.854727843Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Crucible stats registered with UUID: 14ba212f-8a4d-431d-ba70-d7f56c87ec78","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.854752405Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:23.854772747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855262567Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855285959Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910} {"msg":"14ba212f-8a4d-431d-ba70-d7f56c87ec78 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855349505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855375678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"0","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.85539865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"0","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855420893Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"1","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855449076Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"1","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855475598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"2","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855502941Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"2","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855527393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855554256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855580779Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"ds_connection connected from Ok(127.0.0.1:60255)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855610712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"ds_connection connected from Ok(127.0.0.1:64048)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855820503Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"ds_connection connected from Ok(127.0.0.1:50751)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.855867038Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.856421674Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"2","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.856458707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"0","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.856474089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"1","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857099142Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"1","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857131305Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"2","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857158948Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"0","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857707353Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857733506Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[0]R flush_numbers[0..12]: [313, 322, 322, 321, 319, 317, 318, 323, 323, 323, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857788481Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[0]R generation[0..12]: [171, 178, 178, 178, 177, 174, 176, 179, 179, 179, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857806383Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857820355Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[1]R flush_numbers[0..12]: [313, 322, 322, 321, 319, 317, 318, 323, 323, 323, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857833896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[1]R generation[0..12]: [171, 178, 178, 178, 177, 174, 176, 179, 179, 179, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857847547Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857861339Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[2]R flush_numbers[0..12]: [313, 322, 322, 321, 319, 317, 318, 323, 323, 323, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.85787474Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[2]R generation[0..12]: [171, 178, 178, 178, 177, 174, 176, 179, 179, 179, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857888262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857901943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857914424Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:40:23.857930876Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857950778Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857963289Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"":"downstairs","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.85797601Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.857987561Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"0","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.858000373Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"1","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.858013204Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"client":"2","":"downstairs","session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.858026345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.858038346Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"14ba212f-8a4d-431d-ba70-d7f56c87ec78 is now active with session: 9819f899-ceb7-4500-be2f-1ace2ee25501","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.858050928Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.858062519Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910,"session_id":"9819f899-ceb7-4500-be2f-1ace2ee25501"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:23.85807444Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 2 len 36864 data: 7 12 13 13 14 14 16 14 17 03/30 Read block 60 len 24576 04/30 Write block 92 len 8192 data: 13 12 05/30 Write block 314 len 36864 data: 11 14 14 14 15 17 17 21 19 06/30 Read block 563 len 20480 07/30 Read block 363 len 28672 08/30 Read block 52 len 36864 09/30 Write block 159 len 8192 data: 14 11 10/30 Read block 411 len 4096 11/30 Write block 263 len 12288 data: 10 10 10 12/30 Read block 281 len 20480 13/30 Write block 583 len 32768 data: 11 12 10 10 10 10 10 13 14/30 Read block 231 len 16384 15/30 Write block 53 len 16384 data: 13 16 16 15 16/30 Write block 274 len 20480 data: 11 11 11 12 10 17/30 Write block 464 len 40960 data: 13 14 16 16 17 16 16 17 13 10 18/30 Write block 428 len 24576 data: 14 15 15 17 13 10 19/30 Write block 535 len 36864 data: 11 11 11 15 14 15 18 18 17 20/30 Read block 291 len 16384 21/30 Read block 248 len 24576 22/30 Read block 264 len 32768 23/30 Read block 409 len 12288 24/30 Read block 376 len 24576 25/30 Read block 268 len 4096 26/30 Write block 37 len 12288 data: 15 15 15 27/30 Read block 19 len 24576 28/30 Write block 371 len 24576 data: 17 16 15 13 11 8 29/30 Read block 287 len 4096 30/30 Write block 544 len 12288 data: 19 16 15 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:23.878356696Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 313 313 313 T T T 1 020-039 178 178 178 322 322 322 T T T 2 040-059 178 178 178 322 322 322 T T T 3 060-079 178 178 178 321 321 321 F F F 4 080-099 177 177 177 319 319 319 T T T 5 100-119 174 174 174 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 179 179 179 323 323 323 T T T 8 160-179 179 179 179 323 323 323 T T T 9 180-199 179 179 179 323 323 323 F F F 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 178 178 178 320 320 320 T T T 14 280-299 177 177 177 319 319 319 F F F 15 300-319 174 174 174 317 317 317 T T T 16 320-339 173 173 173 316 316 316 T T T 17 340-359 179 179 179 323 323 323 F F F 18 360-379 179 179 179 323 323 323 T T T 19 380-399 176 176 176 318 318 318 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 176 176 176 318 318 318 T T T 22 440-459 179 179 179 323 323 323 F F F 23 460-479 179 179 179 323 323 323 T T T 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 174 174 174 317 317 317 T T T 27 540-559 179 179 179 323 323 323 T T T 28 560-579 178 178 178 321 321 321 F F F 29 580-599 170 170 170 312 312 312 T T T Max gen: 179, Max flush: 323 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 324 324 324 F F F 1 020-039 180 180 180 324 324 324 F F F 2 040-059 180 180 180 324 324 324 F F F 3 060-079 178 178 178 321 321 321 F F F 4 080-099 180 180 180 324 324 324 F F F 5 100-119 174 174 174 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 180 180 180 324 324 324 F F F 8 160-179 180 180 180 324 324 324 F F F 9 180-199 179 179 179 323 323 323 F F F 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 173 173 173 316 316 316 F F F 13 260-279 180 180 180 324 324 324 F F F 14 280-299 177 177 177 319 319 319 F F F 15 300-319 180 180 180 324 324 324 F F F 16 320-339 180 180 180 324 324 324 F F F 17 340-359 179 179 179 323 323 323 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 176 176 176 318 318 318 F F F 20 400-419 176 176 176 318 318 318 F F F 21 420-439 180 180 180 324 324 324 F F F 22 440-459 179 179 179 323 323 323 F F F 23 460-479 180 180 180 324 324 324 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 180 180 180 324 324 324 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 178 178 178 321 321 321 F F F 29 580-599 180 180 180 324 324 324 F F F Max gen: 180, Max flush: 324 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.203488046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:28.246478452Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.246944449Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.278725284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.279552447Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.279595322Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.279617954Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.279636756Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918} {"msg":"Crucible ac02f9f4-eb4a-471e-b376-e72508b80f52 has session id: 79b55bed-30cc-4c2e-8325-4213d82651fa","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.282374712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Upstairs opts: Upstairs UUID: ac02f9f4-eb4a-471e-b376-e72508b80f52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.282402315Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Crucible stats registered with UUID: ac02f9f4-eb4a-471e-b376-e72508b80f52","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.282417106Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:28.282429758Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.282828488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.28285109Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918} {"msg":"ac02f9f4-eb4a-471e-b376-e72508b80f52 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.282924078Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.282956881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"0","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.282987214Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"0","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283004896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"1","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283023048Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"1","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283039319Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"2","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283062841Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"2","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283079333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283097055Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283113637Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:34872)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283275663Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:43355)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283476033Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:56251)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.283526078Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.284097116Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"1","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.284135Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"2","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.284309377Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"0","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.284841031Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"1","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.284964443Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"0","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.284988486Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"2","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285436791Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285556923Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[0]R flush_numbers[0..12]: [324, 324, 324, 321, 324, 317, 318, 324, 324, 323, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285682696Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[0]R generation[0..12]: [180, 180, 180, 178, 180, 174, 176, 180, 180, 179, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285702228Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285716429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[1]R flush_numbers[0..12]: [324, 324, 324, 321, 324, 317, 318, 324, 324, 323, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285745692Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[1]R generation[0..12]: [180, 180, 180, 178, 180, 174, 176, 180, 180, 179, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285759673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285773035Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[2]R flush_numbers[0..12]: [324, 324, 324, 321, 324, 317, 318, 324, 324, 323, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285786566Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[2]R generation[0..12]: [180, 180, 180, 178, 180, 174, 176, 180, 180, 179, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285800068Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.28582083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Max found gen is 181","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285833591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Generation requested: 182 >= found:181","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285846382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285858723Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285871095Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"":"downstairs","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285883966Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285898087Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"0","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285911369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"1","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.28592413Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"client":"2","":"downstairs","session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285936891Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285948872Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ac02f9f4-eb4a-471e-b376-e72508b80f52 is now active with session: 79b55bed-30cc-4c2e-8325-4213d82651fa","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285964524Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285976865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918,"session_id":"79b55bed-30cc-4c2e-8325-4213d82651fa"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:28.285988877Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 405 len 12288 data: 9 10 10 02/30 Flush 03/30 Write block 448 len 12288 data: 10 10 10 04/30 Read block 442 len 36864 05/30 Write block 148 len 12288 data: 9 14 13 06/30 Write block 311 len 24576 data: 14 15 11 12 15 15 07/30 Read block 595 len 16384 08/30 Write block 350 len 36864 data: 15 13 14 12 12 13 14 16 12 09/30 Write block 305 len 20480 data: 14 16 14 15 17 10/30 Write block 102 len 16384 data: 9 12 12 14 11/30 Read block 292 len 40960 12/30 Read block 432 len 12288 13/30 Read block 248 len 4096 14/30 Write block 415 len 20480 data: 15 11 13 16 14 15/30 Read block 70 len 20480 16/30 Read block 18 len 20480 17/30 Read block 411 len 12288 18/30 Write block 44 len 36864 data: 12 11 13 10 9 12 13 15 15 19/30 Read block 5 len 16384 20/30 Read block 409 len 32768 21/30 Write block 458 len 24576 data: 13 11 9 11 11 9 22/30 Read block 72 len 8192 23/30 Flush 24/30 Write block 269 len 4096 data: 10 25/30 Read block 570 len 28672 26/30 Read block 382 len 24576 27/30 Write block 243 len 24576 data: 7 10 13 17 16 16 28/30 Read block 71 len 12288 29/30 Write block 457 len 12288 data: 15 14 12 30/30 Write block 189 len 32768 data: 11 9 13 15 18 17 19 14 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:28.306460231Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 324 324 324 F F F 1 020-039 180 180 180 324 324 324 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 178 178 178 321 321 321 F F F 4 080-099 180 180 180 324 324 324 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 182 182 182 326 326 326 F F F 8 160-179 180 180 180 324 324 324 F F F 9 180-199 179 179 179 323 323 323 T T T 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 173 173 173 316 316 316 T T T 13 260-279 180 180 180 324 324 324 T T T 14 280-299 177 177 177 319 319 319 F F F 15 300-319 182 182 182 326 326 326 F F F 16 320-339 180 180 180 324 324 324 F F F 17 340-359 182 182 182 326 326 326 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 176 176 176 318 318 318 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 180 180 180 324 324 324 F F F 22 440-459 182 182 182 326 326 326 T T T 23 460-479 182 182 182 326 326 326 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 180 180 180 324 324 324 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 178 178 178 321 321 321 F F F 29 580-599 180 180 180 324 324 324 F F F Max gen: 182, Max flush: 326 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 324 324 324 F F F 1 020-039 180 180 180 324 324 324 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 178 178 178 321 321 321 F F F 4 080-099 180 180 180 324 324 324 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 182 182 182 326 326 326 F F F 8 160-179 180 180 180 324 324 324 F F F 9 180-199 183 183 183 327 327 327 F F F 10 200-219 174 174 174 317 317 317 F F F 11 220-239 174 174 174 317 317 317 F F F 12 240-259 183 183 183 327 327 327 F F F 13 260-279 183 183 183 327 327 327 F F F 14 280-299 177 177 177 319 319 319 F F F 15 300-319 182 182 182 326 326 326 F F F 16 320-339 180 180 180 324 324 324 F F F 17 340-359 182 182 182 326 326 326 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 176 176 176 318 318 318 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 180 180 180 324 324 324 F F F 22 440-459 183 183 183 327 327 327 F F F 23 460-479 182 182 182 326 326 326 F F F 24 480-499 171 171 171 313 313 313 F F F 25 500-519 171 171 171 313 313 313 F F F 26 520-539 180 180 180 324 324 324 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 178 178 178 321 321 321 F F F 29 580-599 180 180 180 324 324 324 F F F Max gen: 183, Max flush: 327 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.580062054Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.626007678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.62692579Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.626976506Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.626995588Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.627013529Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926} {"msg":"Crucible 348cbd72-d177-47af-9940-cfbb0d8c5119 has session id: 0929f5fc-36dd-4348-91a1-f80ec6a3864f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.627630001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Upstairs opts: Upstairs UUID: 348cbd72-d177-47af-9940-cfbb0d8c5119, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.627674936Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Crucible stats registered with UUID: 348cbd72-d177-47af-9940-cfbb0d8c5119","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.627705829Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:32.627736832Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628183297Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.62820565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926} {"msg":"348cbd72-d177-47af-9940-cfbb0d8c5119 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628272536Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628295309Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"0","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628320331Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"0","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628334473Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"1","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628348314Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"1","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628369006Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"2","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628394229Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"2","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628418791Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628441173Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628464046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55619)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628632833Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50543)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628681087Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41301)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.628711561Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.629373047Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"1","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.629524233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"0","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.630065637Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"1","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.630107111Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"2","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.630123473Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"0","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.630616073Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.630730464Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.630758707Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"2","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[0]R flush_numbers[0..12]: [324, 324, 326, 321, 324, 326, 318, 326, 324, 327, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631318473Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[0]R generation[0..12]: [180, 180, 182, 178, 180, 182, 176, 182, 180, 183, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631341756Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631356167Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[1]R flush_numbers[0..12]: [324, 324, 326, 321, 324, 326, 318, 326, 324, 327, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631370779Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[1]R generation[0..12]: [180, 180, 182, 178, 180, 182, 176, 182, 180, 183, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.63138452Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631398562Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[2]R flush_numbers[0..12]: [324, 324, 326, 321, 324, 326, 318, 326, 324, 327, 317, 317]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631412103Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[2]R generation[0..12]: [180, 180, 182, 178, 180, 182, 176, 182, 180, 183, 174, 174]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631425434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631442256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631461898Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631475129Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631487651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631500322Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"":"downstairs","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631513383Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631525114Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"0","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631537896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"1","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631550597Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"client":"2","":"downstairs","session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631563438Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.63157529Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"348cbd72-d177-47af-9940-cfbb0d8c5119 is now active with session: 0929f5fc-36dd-4348-91a1-f80ec6a3864f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631587711Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631599522Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926,"session_id":"0929f5fc-36dd-4348-91a1-f80ec6a3864f"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:32.631611483Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 182 len 40960 data: 12 9 13 14 15 13 12 12 10 14 02/30 Read block 224 len 28672 03/30 Write block 243 len 40960 data: 8 11 14 18 17 17 20 21 20 21 04/30 Write block 497 len 28672 data: 10 9 10 8 12 13 12 05/30 Write block 188 len 8192 data: 13 13 06/30 Read block 562 len 40960 07/30 Flush 08/30 Write block 524 len 32768 data: 13 14 17 17 16 18 16 17 09/30 Read block 142 len 20480 10/30 Flush 11/30 Write block 241 len 12288 data: 11 8 9 12/30 Flush 13/30 Write block 502 len 24576 data: 14 13 12 14 13 9 14/30 Read block 239 len 16384 15/30 Write block 275 len 32768 data: 12 12 13 11 12 10 8 5 16/30 Read block 365 len 8192 17/30 Write block 515 len 4096 data: 15 18/30 Read block 555 len 8192 19/30 Read block 166 len 12288 20/30 Write block 475 len 16384 data: 9 10 7 6 21/30 Write block 162 len 32768 data: 12 15 13 14 13 13 14 14 22/30 Write block 188 len 16384 data: 14 14 11 15 23/30 Write block 483 len 36864 data: 9 10 12 10 13 16 14 11 12 24/30 Read block 100 len 8192 25/30 Write block 544 len 28672 data: 20 17 16 16 22 24 24 26/30 Read block 561 len 4096 27/30 Write block 208 len 8192 data: 10 12 28/30 Read block 506 len 36864 29/30 Read block 249 len 16384 30/30 Write block 234 len 32768 data: 16 20 21 19 18 17 13 12 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 6 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:32.651764046Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 324 324 324 F F F 1 020-039 180 180 180 324 324 324 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 178 178 178 321 321 321 F F F 4 080-099 180 180 180 324 324 324 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 182 182 182 326 326 326 F F F 8 160-179 180 180 180 324 324 324 T T T 9 180-199 184 184 184 328 328 328 T T T 10 200-219 174 174 174 317 317 317 T T T 11 220-239 174 174 174 317 317 317 T T T 12 240-259 184 184 184 330 330 330 T T T 13 260-279 183 183 183 327 327 327 T T T 14 280-299 177 177 177 319 319 319 T T T 15 300-319 182 182 182 326 326 326 F F F 16 320-339 180 180 180 324 324 324 F F F 17 340-359 182 182 182 326 326 326 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 176 176 176 318 318 318 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 180 180 180 324 324 324 F F F 22 440-459 183 183 183 327 327 327 F F F 23 460-479 182 182 182 326 326 326 T T T 24 480-499 184 184 184 328 328 328 T T T 25 500-519 184 184 184 328 328 328 T T T 26 520-539 184 184 184 329 329 329 F F F 27 540-559 180 180 180 324 324 324 T T T 28 560-579 178 178 178 321 321 321 F F F 29 580-599 180 180 180 324 324 324 F F F Max gen: 184, Max flush: 330 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 324 324 324 F F F 1 020-039 180 180 180 324 324 324 F F F 2 040-059 182 182 182 326 326 326 F F F 3 060-079 178 178 178 321 321 321 F F F 4 080-099 180 180 180 324 324 324 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 182 182 182 326 326 326 F F F 8 160-179 185 185 185 331 331 331 F F F 9 180-199 185 185 185 331 331 331 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 185 185 185 331 331 331 F F F 12 240-259 185 185 185 331 331 331 F F F 13 260-279 185 185 185 331 331 331 F F F 14 280-299 185 185 185 331 331 331 F F F 15 300-319 182 182 182 326 326 326 F F F 16 320-339 180 180 180 324 324 324 F F F 17 340-359 182 182 182 326 326 326 F F F 18 360-379 180 180 180 324 324 324 F F F 19 380-399 176 176 176 318 318 318 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 180 180 180 324 324 324 F F F 22 440-459 183 183 183 327 327 327 F F F 23 460-479 185 185 185 331 331 331 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 185 185 185 331 331 331 F F F 26 520-539 184 184 184 329 329 329 F F F 27 540-559 185 185 185 331 331 331 F F F 28 560-579 178 178 178 321 321 321 F F F 29 580-599 180 180 180 324 324 324 F F F Max gen: 185, Max flush: 331 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.930224946Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.975934636Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.976912545Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.97695972Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.976974651Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.976989243Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934} {"msg":"Crucible 8179c9db-e9bb-4b7b-b9dd-5294782ff081 has session id: bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.977729197Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Upstairs opts: Upstairs UUID: 8179c9db-e9bb-4b7b-b9dd-5294782ff081, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.977765361Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Crucible stats registered with UUID: 8179c9db-e9bb-4b7b-b9dd-5294782ff081","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.977781543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:46.977794014Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978275082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978297875Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934} {"msg":"8179c9db-e9bb-4b7b-b9dd-5294782ff081 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978394074Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978409736Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"0","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978432838Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"0","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.97844754Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"1","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978471532Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"1","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978496015Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"2","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978520237Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"2","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978540909Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978564462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978610246Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45640)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978717137Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61840)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978837209Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50233)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.978874233Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.9794438Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"2","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.979470703Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"0","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.980091125Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"1","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.980129239Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"2","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.980159382Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"0","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.980696866Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.980811278Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"1","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.98082953Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[0]R flush_numbers[0..12]: [324, 324, 326, 321, 324, 326, 318, 326, 331, 331, 331, 331]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981395327Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[0]R generation[0..12]: [180, 180, 182, 178, 180, 182, 176, 182, 185, 185, 185, 185]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981419179Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981436721Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[1]R flush_numbers[0..12]: [324, 324, 326, 321, 324, 326, 318, 326, 331, 331, 331, 331]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981451823Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[1]R generation[0..12]: [180, 180, 182, 178, 180, 182, 176, 182, 185, 185, 185, 185]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981465804Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981479956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[2]R flush_numbers[0..12]: [324, 324, 326, 321, 324, 326, 318, 326, 331, 331, 331, 331]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981493397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[2]R generation[0..12]: [180, 180, 182, 178, 180, 182, 176, 182, 185, 185, 185, 185]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981506598Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.98152089Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981540832Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981554423Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981567114Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981579826Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"":"downstairs","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981592407Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981603798Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"0","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981616729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"1","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981631081Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"client":"2","":"downstairs","session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981644012Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981655803Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"8179c9db-e9bb-4b7b-b9dd-5294782ff081 is now active with session: bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981672455Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981684286Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934,"session_id":"bbc1ed1f-fdb3-45c4-8971-ed0b47d1a44f"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:40:46.981696927Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 583 len 36864 data: 12 13 11 11 11 11 11 14 14 02/30 Write block 128 len 40960 data: 8 7 8 8 11 13 15 13 14 14 03/30 Write block 75 len 40960 data: 18 21 15 15 15 16 12 13 14 13 04/30 Read block 200 len 8192 05/30 Write block 49 len 16384 data: 13 14 16 16 06/30 Write block 448 len 16384 data: 11 11 11 9 07/30 Write block 48 len 16384 data: 10 14 15 17 08/30 Write block 336 len 16384 data: 20 14 13 10 09/30 Write block 447 len 20480 data: 10 12 12 12 10 10/30 Write block 508 len 24576 data: 9 14 15 11 13 15 11/30 Read block 216 len 32768 12/30 Read block 239 len 40960 13/30 Read block 137 len 12288 14/30 Read block 143 len 20480 15/30 Read block 408 len 28672 16/30 Write block 307 len 32768 data: 15 16 18 16 15 16 12 13 17/30 Flush 18/30 Flush 19/30 Write block 271 len 16384 data: 9 10 10 12 20/30 Read block 201 len 32768 21/30 Read block 311 len 4096 22/30 Write block 516 len 20480 data: 14 12 13 16 15 23/30 Read block 92 len 16384 24/30 Write block 192 len 12288 data: 16 19 18 25/30 Write block 556 len 4096 data: 14 26/30 Write block 265 len 24576 data: 11 10 13 13 11 10 27/30 Write block 377 len 24576 data: 10 11 12 10 11 14 28/30 Read block 232 len 4096 29/30 Read block 418 len 4096 30/30 Read block 453 len 12288 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:40:47.000569081Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 324 324 324 F F F 1 020-039 180 180 180 324 324 324 F F F 2 040-059 186 186 186 332 332 332 F F F 3 060-079 186 186 186 332 332 332 F F F 4 080-099 186 186 186 332 332 332 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 186 186 186 332 332 332 F F F 7 140-159 182 182 182 326 326 326 F F F 8 160-179 185 185 185 331 331 331 F F F 9 180-199 185 185 185 331 331 331 T T T 10 200-219 185 185 185 331 331 331 F F F 11 220-239 185 185 185 331 331 331 F F F 12 240-259 185 185 185 331 331 331 F F F 13 260-279 185 185 185 331 331 331 T T T 14 280-299 185 185 185 331 331 331 F F F 15 300-319 186 186 186 332 332 332 F F F 16 320-339 186 186 186 332 332 332 F F F 17 340-359 182 182 182 326 326 326 F F F 18 360-379 180 180 180 324 324 324 T T T 19 380-399 176 176 176 318 318 318 T T T 20 400-419 182 182 182 326 326 326 F F F 21 420-439 180 180 180 324 324 324 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 185 185 185 331 331 331 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 186 186 186 332 332 332 T T T 26 520-539 184 184 184 329 329 329 T T T 27 540-559 185 185 185 331 331 331 T T T 28 560-579 178 178 178 321 321 321 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 186, Max flush: 332 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 324 324 324 F F F 1 020-039 180 180 180 324 324 324 F F F 2 040-059 186 186 186 332 332 332 F F F 3 060-079 186 186 186 332 332 332 F F F 4 080-099 186 186 186 332 332 332 F F F 5 100-119 182 182 182 326 326 326 F F F 6 120-139 186 186 186 332 332 332 F F F 7 140-159 182 182 182 326 326 326 F F F 8 160-179 185 185 185 331 331 331 F F F 9 180-199 187 187 187 333 333 333 F F F 10 200-219 185 185 185 331 331 331 F F F 11 220-239 185 185 185 331 331 331 F F F 12 240-259 185 185 185 331 331 331 F F F 13 260-279 187 187 187 333 333 333 F F F 14 280-299 185 185 185 331 331 331 F F F 15 300-319 186 186 186 332 332 332 F F F 16 320-339 186 186 186 332 332 332 F F F 17 340-359 182 182 182 326 326 326 F F F 18 360-379 187 187 187 333 333 333 F F F 19 380-399 187 187 187 333 333 333 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 180 180 180 324 324 324 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 185 185 185 331 331 331 F F F 24 480-499 185 185 185 331 331 331 F F F 25 500-519 187 187 187 333 333 333 F F F 26 520-539 187 187 187 333 333 333 F F F 27 540-559 187 187 187 333 333 333 F F F 28 560-579 178 178 178 321 321 321 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 187, Max flush: 333 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.256450026Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.302222943Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.303079549Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.303129874Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.303149216Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.303167188Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942} {"msg":"Crucible 90e2b786-78d9-414d-a5e7-78520988c4af has session id: 9e10ec97-ef0e-4492-bfd4-47e3bd00a05b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.303623064Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Upstairs opts: Upstairs UUID: 90e2b786-78d9-414d-a5e7-78520988c4af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.303664678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Crucible stats registered with UUID: 90e2b786-78d9-414d-a5e7-78520988c4af","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.303689241Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:01.303709523Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304192522Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304225345Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942} {"msg":"90e2b786-78d9-414d-a5e7-78520988c4af active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304308533Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304342346Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"0","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304392712Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"0","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304420505Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"1","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304447527Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"1","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304490912Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"2","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304522085Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"2","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304549807Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.30457621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304600283Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"ds_connection connected from Ok(127.0.0.1:52478)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304627865Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56394)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304660419Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49594)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.304703333Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.305643658Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"0","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.305804494Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"1","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.306224746Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"0","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.306375682Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"1","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.306411925Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"2","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.306781603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.306904285Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307163671Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"2","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[0]R flush_numbers[0..12]: [324, 324, 332, 332, 332, 326, 332, 326, 331, 333, 331, 331]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307680153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[0]R generation[0..12]: [180, 180, 186, 186, 186, 182, 186, 182, 185, 187, 185, 185]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307704866Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307720377Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[1]R flush_numbers[0..12]: [324, 324, 332, 332, 332, 326, 332, 326, 331, 333, 331, 331]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307734279Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[1]R generation[0..12]: [180, 180, 186, 186, 186, 182, 186, 182, 185, 187, 185, 185]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.30774765Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307761491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[2]R flush_numbers[0..12]: [324, 324, 332, 332, 332, 326, 332, 326, 331, 333, 331, 331]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307775083Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[2]R generation[0..12]: [180, 180, 186, 186, 186, 182, 186, 182, 185, 187, 185, 185]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307788364Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307802075Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307825118Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307838839Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.30785172Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307864582Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"":"downstairs","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307877663Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307889464Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"0","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307902406Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"1","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307914997Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"client":"2","":"downstairs","session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307927678Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307939709Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"90e2b786-78d9-414d-a5e7-78520988c4af is now active with session: 9e10ec97-ef0e-4492-bfd4-47e3bd00a05b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307952281Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:01.307963792Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942,"session_id":"9e10ec97-ef0e-4492-bfd4-47e3bd00a05b"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"level":30,"time":"2025-02-27T17:41:01.307979323Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 296 len 4096 data: 11 02/30 Read block 582 len 24576 03/30 Read block 477 len 4096 04/30 Read block 567 len 32768 05/30 Read block 49 len 28672 06/30 Write block 423 len 8192 data: 14 11 07/30 Flush 08/30 Write block 518 len 12288 data: 14 17 16 09/30 Write block 6 len 32768 data: 15 15 17 15 18 14 16 18 10/30 Read block 3 len 8192 11/30 Write block 472 len 28672 data: 14 11 9 10 11 8 7 12/30 Read block 179 len 8192 13/30 Write block 114 len 32768 data: 10 8 13 15 17 16 22 22 14/30 Read block 131 len 4096 15/30 Read block 142 len 4096 16/30 Write block 212 len 40960 data: 16 17 15 18 20 18 18 19 18 18 17/30 Flush 18/30 Write block 512 len 20480 data: 14 16 16 16 15 19/30 Write block 33 len 12288 data: 14 13 13 20/30 Write block 479 len 28672 data: 4 6 8 12 10 11 13 21/30 Write block 121 len 28672 data: 23 19 17 17 15 11 10 22/30 Flush 23/30 Write block 561 len 8192 data: 10 7 24/30 Write block 150 len 16384 data: 14 14 14 15 25/30 Read block 498 len 8192 26/30 Flush 27/30 Write block 348 len 12288 data: 13 12 16 28/30 Read block 50 len 20480 29/30 Read block 329 len 4096 30/30 Read block 480 len 16384 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 3 4 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:01.32638564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 335 335 335 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 186 186 186 332 332 332 F F F 3 060-079 186 186 186 332 332 332 F F F 4 080-099 186 186 186 332 332 332 F F F 5 100-119 188 188 188 335 335 335 F F F 6 120-139 188 188 188 336 336 336 F F F 7 140-159 188 188 188 337 337 337 F F F 8 160-179 185 185 185 331 331 331 F F F 9 180-199 187 187 187 333 333 333 F F F 10 200-219 188 188 188 335 335 335 F F F 11 220-239 188 188 188 335 335 335 F F F 12 240-259 185 185 185 331 331 331 F F F 13 260-279 187 187 187 333 333 333 F F F 14 280-299 188 188 188 334 334 334 F F F 15 300-319 186 186 186 332 332 332 F F F 16 320-339 186 186 186 332 332 332 F F F 17 340-359 182 182 182 326 326 326 T T T 18 360-379 187 187 187 333 333 333 F F F 19 380-399 187 187 187 333 333 333 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 188 188 188 334 334 334 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 188 188 188 336 336 336 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 188 188 188 336 336 336 F F F 26 520-539 188 188 188 335 335 335 F F F 27 540-559 187 187 187 333 333 333 F F F 28 560-579 188 188 188 337 337 337 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 188, Max flush: 337 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 335 335 335 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 186 186 186 332 332 332 F F F 3 060-079 186 186 186 332 332 332 F F F 4 080-099 186 186 186 332 332 332 F F F 5 100-119 188 188 188 335 335 335 F F F 6 120-139 188 188 188 336 336 336 F F F 7 140-159 188 188 188 337 337 337 F F F 8 160-179 185 185 185 331 331 331 F F F 9 180-199 187 187 187 333 333 333 F F F 10 200-219 188 188 188 335 335 335 F F F 11 220-239 188 188 188 335 335 335 F F F 12 240-259 185 185 185 331 331 331 F F F 13 260-279 187 187 187 333 333 333 F F F 14 280-299 188 188 188 334 334 334 F F F 15 300-319 186 186 186 332 332 332 F F F 16 320-339 186 186 186 332 332 332 F F F 17 340-359 189 189 189 338 338 338 F F F 18 360-379 187 187 187 333 333 333 F F F 19 380-399 187 187 187 333 333 333 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 188 188 188 334 334 334 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 188 188 188 336 336 336 F F F 24 480-499 188 188 188 336 336 336 F F F 25 500-519 188 188 188 336 336 336 F F F 26 520-539 188 188 188 335 335 335 F F F 27 540-559 187 187 187 333 333 333 F F F 28 560-579 188 188 188 337 337 337 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 189, Max flush: 338 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.550723723Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.594872896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.595602279Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.595648554Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.595665986Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.595680637Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950} {"msg":"Crucible 2e634041-15ee-47ef-9739-64c7918e706a has session id: d0e80c10-843f-4465-93c3-606625f6961f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596216661Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Upstairs opts: Upstairs UUID: 2e634041-15ee-47ef-9739-64c7918e706a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596259696Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Crucible stats registered with UUID: 2e634041-15ee-47ef-9739-64c7918e706a","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596276747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:15.596293729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596752295Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596776248Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950} {"msg":"2e634041-15ee-47ef-9739-64c7918e706a active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596840884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596857286Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"0","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596880708Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"0","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.5969018Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596927743Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596952786Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"2","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.596979238Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"2","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.597005991Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.597032223Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.597054316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63535)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.597077468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59531)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.59710099Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:15.597223583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:15.59729209Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:15.597329013Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.597444815Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.597949636Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"2","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.597975849Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"0","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.598594621Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"0","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.598621474Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"2","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.599182411Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:15.599207393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.597809122Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62238)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.59789153Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.599102862Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.599735466Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[0]R flush_numbers[0..12]: [335, 336, 332, 332, 332, 335, 336, 337, 331, 333, 335, 335]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600396113Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[0]R generation[0..12]: [188, 188, 186, 186, 186, 188, 188, 188, 185, 187, 188, 188]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600430616Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600450488Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[1]R flush_numbers[0..12]: [335, 336, 332, 332, 332, 335, 336, 337, 331, 333, 335, 335]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.60046795Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[1]R generation[0..12]: [188, 188, 186, 186, 186, 188, 188, 188, 185, 187, 188, 188]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600485072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600502603Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[2]R flush_numbers[0..12]: [335, 336, 332, 332, 332, 335, 336, 337, 331, 333, 335, 335]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600519725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[2]R generation[0..12]: [188, 188, 186, 186, 186, 188, 188, 188, 185, 187, 188, 188]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:41:25.600536777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600558709Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600574571Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600590442Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600605994Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600621515Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"":"downstairs","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600637527Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600656969Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"0","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600673731Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"1","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600690262Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"client":"2","":"downstairs","session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600706834Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600721876Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"2e634041-15ee-47ef-9739-64c7918e706a is now active with session: d0e80c10-843f-4465-93c3-606625f6961f","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600737357Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.600751828Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950,"session_id":"d0e80c10-843f-4465-93c3-606625f6961f"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:25.60076664Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 533 len 32768 data: 13 11 12 12 12 16 15 16 02/30 Write block 52 len 24576 data: 17 14 17 17 16 15 03/30 Write block 239 len 32768 data: 18 14 13 9 10 12 15 19 04/30 Read block 44 len 40960 05/30 Read block 428 len 28672 06/30 Flush 07/30 Write block 543 len 12288 data: 18 21 18 08/30 Read block 569 len 40960 09/30 Write block 317 len 40960 data: 15 16 18 18 22 20 20 19 17 16 10/30 Write block 177 len 28672 data: 10 8 11 11 13 13 10 11/30 Read block 559 len 8192 12/30 Write block 368 len 12288 data: 14 15 14 13/30 Read block 421 len 16384 14/30 Write block 77 len 4096 data: 16 15/30 Read block 115 len 24576 16/30 Read block 116 len 8192 17/30 Write block 239 len 32768 data: 19 15 14 10 11 13 16 20 18/30 Read block 494 len 36864 19/30 Read block 513 len 24576 20/30 Write block 66 len 24576 data: 14 16 15 16 18 19 21/30 Read block 336 len 32768 22/30 Write block 90 len 8192 data: 16 13 23/30 Read block 47 len 20480 24/30 Read block 543 len 32768 25/30 Write block 394 len 12288 data: 11 13 14 26/30 Write block 11 len 32768 data: 15 17 19 18 20 17 14 12 27/30 Write block 492 len 16384 data: 11 10 11 10 28/30 Write block 390 len 8192 data: 9 13 29/30 Read block 453 len 12288 30/30 Read block 587 len 8192 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 3 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:25.620701381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 335 335 335 T T T 1 020-039 188 188 188 336 336 336 F F F 2 040-059 190 190 190 339 339 339 F F F 3 060-079 186 186 186 332 332 332 T T T 4 080-099 186 186 186 332 332 332 T T T 5 100-119 188 188 188 335 335 335 F F F 6 120-139 188 188 188 336 336 336 F F F 7 140-159 188 188 188 337 337 337 F F F 8 160-179 185 185 185 331 331 331 T T T 9 180-199 187 187 187 333 333 333 T T T 10 200-219 188 188 188 335 335 335 F F F 11 220-239 190 190 190 339 339 339 T T T 12 240-259 190 190 190 339 339 339 T T T 13 260-279 187 187 187 333 333 333 F F F 14 280-299 188 188 188 334 334 334 F F F 15 300-319 186 186 186 332 332 332 T T T 16 320-339 186 186 186 332 332 332 T T T 17 340-359 189 189 189 338 338 338 F F F 18 360-379 187 187 187 333 333 333 T T T 19 380-399 187 187 187 333 333 333 T T T 20 400-419 182 182 182 326 326 326 F F F 21 420-439 188 188 188 334 334 334 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 188 188 188 336 336 336 F F F 24 480-499 188 188 188 336 336 336 T T T 25 500-519 188 188 188 336 336 336 F F F 26 520-539 190 190 190 339 339 339 F F F 27 540-559 190 190 190 339 339 339 T T T 28 560-579 188 188 188 337 337 337 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 190, Max flush: 339 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 340 340 340 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 190 190 190 339 339 339 F F F 3 060-079 191 191 191 340 340 340 F F F 4 080-099 191 191 191 340 340 340 F F F 5 100-119 188 188 188 335 335 335 F F F 6 120-139 188 188 188 336 336 336 F F F 7 140-159 188 188 188 337 337 337 F F F 8 160-179 191 191 191 340 340 340 F F F 9 180-199 191 191 191 340 340 340 F F F 10 200-219 188 188 188 335 335 335 F F F 11 220-239 191 191 191 340 340 340 F F F 12 240-259 191 191 191 340 340 340 F F F 13 260-279 187 187 187 333 333 333 F F F 14 280-299 188 188 188 334 334 334 F F F 15 300-319 191 191 191 340 340 340 F F F 16 320-339 191 191 191 340 340 340 F F F 17 340-359 189 189 189 338 338 338 F F F 18 360-379 191 191 191 340 340 340 F F F 19 380-399 191 191 191 340 340 340 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 188 188 188 334 334 334 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 188 188 188 336 336 336 F F F 24 480-499 191 191 191 340 340 340 F F F 25 500-519 188 188 188 336 336 336 F F F 26 520-539 190 190 190 339 339 339 F F F 27 540-559 191 191 191 340 340 340 F F F 28 560-579 188 188 188 337 337 337 F F F 29 580-599 186 186 186 332 332 332 F F F Max gen: 191, Max flush: 340 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.904353516Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.948578316Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.949385247Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.949428092Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.949443884Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.949458475Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958} {"msg":"Crucible df4ef9b9-5315-4ed5-82df-72a75bd03ecf has session id: 5b456da2-ce85-45f4-b49b-0bf63102120b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950015341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Upstairs opts: Upstairs UUID: df4ef9b9-5315-4ed5-82df-72a75bd03ecf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950075397Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Crucible stats registered with UUID: df4ef9b9-5315-4ed5-82df-72a75bd03ecf","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950093879Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:39.950110491Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950607701Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950636284Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958} {"msg":"df4ef9b9-5315-4ed5-82df-72a75bd03ecf active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950717862Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950738574Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"0","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950781388Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"0","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950809361Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950837994Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950868887Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"2","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.95089899Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"2","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950927253Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950954526Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.950980699Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43282)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.951004021Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:39.951050386Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:39.951083459Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:39.951112762Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65429)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.951204171Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.951236354Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.951718573Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"0","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.951757417Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"2","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.952427925Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"0","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.952455797Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"2","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.953169769Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:39.953200112Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.951661396Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45149)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.951804921Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.953005072Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.953595611Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[0]R flush_numbers[0..12]: [340, 336, 339, 340, 340, 335, 336, 337, 340, 340, 335, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954226765Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[0]R generation[0..12]: [191, 188, 190, 191, 191, 188, 188, 188, 191, 191, 188, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954259418Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.95427572Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[1]R flush_numbers[0..12]: [340, 336, 339, 340, 340, 335, 336, 337, 340, 340, 335, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954290462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[1]R generation[0..12]: [191, 188, 190, 191, 191, 188, 188, 188, 191, 191, 188, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954304273Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954326935Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[2]R flush_numbers[0..12]: [340, 336, 339, 340, 340, 335, 336, 337, 340, 340, 335, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954347537Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[2]R generation[0..12]: [191, 188, 190, 191, 191, 188, 188, 188, 191, 191, 188, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954361629Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.95437511Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954387751Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-02-27T17:41:49.954400593Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954414934Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954427915Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"":"downstairs","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954440817Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954452468Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"0","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954467689Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"1","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954480591Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"client":"2","":"downstairs","session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954493362Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954505063Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"df4ef9b9-5315-4ed5-82df-72a75bd03ecf is now active with session: 5b456da2-ce85-45f4-b49b-0bf63102120b","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954517744Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954529435Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958,"session_id":"5b456da2-ce85-45f4-b49b-0bf63102120b"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:49.954541157Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 161 len 12288 data: 11 13 16 02/30 Read block 512 len 12288 03/30 Read block 236 len 8192 04/30 Read block 363 len 16384 05/30 Read block 270 len 40960 06/30 Write block 202 len 16384 data: 14 12 11 12 07/30 Read block 228 len 4096 08/30 Read block 329 len 12288 09/30 Write block 192 len 8192 data: 17 20 10/30 Write block 98 len 20480 data: 9 10 11 11 10 11/30 Flush 12/30 Write block 394 len 20480 data: 12 14 15 12 9 13/30 Write block 338 len 32768 data: 14 11 10 9 11 12 7 7 14/30 Read block 68 len 24576 15/30 Read block 495 len 20480 16/30 Write block 424 len 20480 data: 12 15 19 16 15 17/30 Read block 309 len 12288 18/30 Read block 40 len 16384 19/30 Write block 542 len 20480 data: 19 19 22 19 17 20/30 Read block 135 len 28672 21/30 Read block 387 len 4096 22/30 Write block 294 len 4096 data: 10 23/30 Write block 577 len 32768 data: 12 12 11 8 9 11 13 14 24/30 Write block 255 len 8192 data: 14 14 25/30 Read block 563 len 16384 26/30 Read block 168 len 16384 27/30 Read block 494 len 36864 28/30 Read block 242 len 8192 29/30 Write block 470 len 24576 data: 17 18 15 12 10 11 30/30 Read block 354 len 16384 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 5 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 18 19 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:49.974390199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 340 340 340 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 190 190 190 339 339 339 F F F 3 060-079 191 191 191 340 340 340 F F F 4 080-099 192 192 192 341 341 341 F F F 5 100-119 192 192 192 341 341 341 F F F 6 120-139 188 188 188 336 336 336 F F F 7 140-159 188 188 188 337 337 337 F F F 8 160-179 192 192 192 341 341 341 F F F 9 180-199 192 192 192 341 341 341 F F F 10 200-219 192 192 192 341 341 341 F F F 11 220-239 191 191 191 340 340 340 F F F 12 240-259 191 191 191 340 340 340 T T T 13 260-279 187 187 187 333 333 333 F F F 14 280-299 188 188 188 334 334 334 T T T 15 300-319 191 191 191 340 340 340 F F F 16 320-339 191 191 191 340 340 340 T T T 17 340-359 189 189 189 338 338 338 T T T 18 360-379 191 191 191 340 340 340 F F F 19 380-399 191 191 191 340 340 340 T T T 20 400-419 182 182 182 326 326 326 F F F 21 420-439 188 188 188 334 334 334 T T T 22 440-459 186 186 186 332 332 332 F F F 23 460-479 188 188 188 336 336 336 T T T 24 480-499 191 191 191 340 340 340 F F F 25 500-519 188 188 188 336 336 336 F F F 26 520-539 190 190 190 339 339 339 F F F 27 540-559 191 191 191 340 340 340 T T T 28 560-579 188 188 188 337 337 337 T T T 29 580-599 186 186 186 332 332 332 T T T Max gen: 192, Max flush: 341 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 340 340 340 F F F 1 020-039 188 188 188 336 336 336 F F F 2 040-059 190 190 190 339 339 339 F F F 3 060-079 191 191 191 340 340 340 F F F 4 080-099 192 192 192 341 341 341 F F F 5 100-119 192 192 192 341 341 341 F F F 6 120-139 188 188 188 336 336 336 F F F 7 140-159 188 188 188 337 337 337 F F F 8 160-179 192 192 192 341 341 341 F F F 9 180-199 192 192 192 341 341 341 F F F 10 200-219 192 192 192 341 341 341 F F F 11 220-239 191 191 191 340 340 340 F F F 12 240-259 193 193 193 342 342 342 F F F 13 260-279 187 187 187 333 333 333 F F F 14 280-299 193 193 193 342 342 342 F F F 15 300-319 191 191 191 340 340 340 F F F 16 320-339 193 193 193 342 342 342 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 191 191 191 340 340 340 F F F 19 380-399 193 193 193 342 342 342 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 193 193 193 342 342 342 F F F 22 440-459 186 186 186 332 332 332 F F F 23 460-479 193 193 193 342 342 342 F F F 24 480-499 191 191 191 340 340 340 F F F 25 500-519 188 188 188 336 336 336 F F F 26 520-539 190 190 190 339 339 339 F F F 27 540-559 193 193 193 342 342 342 F F F 28 560-579 193 193 193 342 342 342 F F F 29 580-599 193 193 193 342 342 342 F F F Max gen: 193, Max flush: 342 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.247823874Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.293121453Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.293858757Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.293899261Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.293913843Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.293928024Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966} {"msg":"Crucible e4031d3e-3059-4c34-9556-07cb569ad1d7 has session id: d1be7156-6666-4e48-816c-9fff86bf6374","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.294611573Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Upstairs opts: Upstairs UUID: e4031d3e-3059-4c34-9556-07cb569ad1d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.294653777Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Crucible stats registered with UUID: e4031d3e-3059-4c34-9556-07cb569ad1d7","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.294676729Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:54.294696462Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.2951766Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295210583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966} {"msg":"e4031d3e-3059-4c34-9556-07cb569ad1d7 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.29527148Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295296562Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"0","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295322895Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"0","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295340796Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"1","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295360629Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"1","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.2953745Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"2","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295388471Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"2","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295401343Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295465069Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295491892Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"ds_connection connected from Ok(127.0.0.1:53423)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295522995Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"ds_connection connected from Ok(127.0.0.1:38742)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295562819Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"ds_connection connected from Ok(127.0.0.1:38560)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.295590942Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.296421916Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"2","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.296457669Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"0","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.297110765Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"2","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.297149989Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"0","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.297802075Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.297826307Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.298083173Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"1","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.298583834Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"1","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[0]R flush_numbers[0..12]: [340, 336, 339, 340, 341, 341, 336, 337, 341, 341, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299116647Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[0]R generation[0..12]: [191, 188, 190, 191, 192, 192, 188, 188, 192, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299135559Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299149881Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[1]R flush_numbers[0..12]: [340, 336, 339, 340, 341, 341, 336, 337, 341, 341, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299164782Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[1]R generation[0..12]: [191, 188, 190, 191, 192, 192, 188, 188, 192, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299180714Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299197545Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[2]R flush_numbers[0..12]: [340, 336, 339, 340, 341, 341, 336, 337, 341, 341, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299214057Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[2]R generation[0..12]: [191, 188, 190, 191, 192, 192, 188, 188, 192, 192, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299234389Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299251521Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299273423Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299289525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Next flush: 343","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299305236Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299320978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"":"downstairs","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.29933705Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299351801Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"0","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299367713Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"1","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299383564Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"client":"2","":"downstairs","session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299399566Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299414247Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"e4031d3e-3059-4c34-9556-07cb569ad1d7 is now active with session: d1be7156-6666-4e48-816c-9fff86bf6374","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299429849Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299448331Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966,"session_id":"d1be7156-6666-4e48-816c-9fff86bf6374"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:41:54.299463132Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 509 len 36864 data: 15 16 12 15 17 17 17 16 13 02/30 Read block 19 len 12288 03/30 Read block 438 len 36864 04/30 Read block 583 len 28672 05/30 Read block 477 len 16384 06/30 Read block 106 len 16384 07/30 Read block 7 len 4096 08/30 Read block 538 len 40960 09/30 Read block 531 len 28672 10/30 Write block 249 len 12288 data: 21 22 21 11/30 Read block 480 len 20480 12/30 Flush 13/30 Write block 102 len 24576 data: 11 13 13 15 14 13 14/30 Read block 358 len 36864 15/30 Flush 16/30 Write block 249 len 36864 data: 22 23 22 22 24 19 15 15 14 17/30 Read block 309 len 36864 18/30 Write block 14 len 32768 data: 19 21 18 15 13 12 11 11 19/30 Write block 178 len 16384 data: 9 12 12 14 20/30 Write block 136 len 4096 data: 15 21/30 Read block 264 len 36864 22/30 Write block 240 len 40960 data: 16 15 11 12 14 17 21 18 18 23 23/30 Write block 263 len 24576 data: 11 11 12 11 14 14 24/30 Read block 334 len 16384 25/30 Write block 454 len 40960 data: 12 12 14 16 15 13 10 12 12 10 26/30 Read block 568 len 28672 27/30 Read block 227 len 24576 28/30 Read block 21 len 24576 29/30 Write block 515 len 28672 data: 18 17 14 15 18 17 16 30/30 Write block 150 len 20480 data: 15 15 15 16 15 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:41:54.321908916Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 340 340 340 T T T 1 020-039 188 188 188 336 336 336 T T T 2 040-059 190 190 190 339 339 339 F F F 3 060-079 191 191 191 340 340 340 F F F 4 080-099 192 192 192 341 341 341 F F F 5 100-119 194 194 194 344 344 344 F F F 6 120-139 188 188 188 336 336 336 T T T 7 140-159 188 188 188 337 337 337 T T T 8 160-179 192 192 192 341 341 341 T T T 9 180-199 192 192 192 341 341 341 T T T 10 200-219 192 192 192 341 341 341 F F F 11 220-239 191 191 191 340 340 340 F F F 12 240-259 194 194 194 343 343 343 T T T 13 260-279 187 187 187 333 333 333 T T T 14 280-299 193 193 193 342 342 342 F F F 15 300-319 191 191 191 340 340 340 F F F 16 320-339 193 193 193 342 342 342 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 191 191 191 340 340 340 F F F 19 380-399 193 193 193 342 342 342 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 193 193 193 342 342 342 F F F 22 440-459 186 186 186 332 332 332 T T T 23 460-479 193 193 193 342 342 342 T T T 24 480-499 191 191 191 340 340 340 F F F 25 500-519 194 194 194 343 343 343 T T T 26 520-539 190 190 190 339 339 339 T T T 27 540-559 193 193 193 342 342 342 F F F 28 560-579 193 193 193 342 342 342 F F F 29 580-599 193 193 193 342 342 342 F F F Max gen: 194, Max flush: 344 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 345 345 345 F F F 1 020-039 195 195 195 345 345 345 F F F 2 040-059 190 190 190 339 339 339 F F F 3 060-079 191 191 191 340 340 340 F F F 4 080-099 192 192 192 341 341 341 F F F 5 100-119 194 194 194 344 344 344 F F F 6 120-139 195 195 195 345 345 345 F F F 7 140-159 195 195 195 345 345 345 F F F 8 160-179 195 195 195 345 345 345 F F F 9 180-199 195 195 195 345 345 345 F F F 10 200-219 192 192 192 341 341 341 F F F 11 220-239 191 191 191 340 340 340 F F F 12 240-259 195 195 195 345 345 345 F F F 13 260-279 195 195 195 345 345 345 F F F 14 280-299 193 193 193 342 342 342 F F F 15 300-319 191 191 191 340 340 340 F F F 16 320-339 193 193 193 342 342 342 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 191 191 191 340 340 340 F F F 19 380-399 193 193 193 342 342 342 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 193 193 193 342 342 342 F F F 22 440-459 195 195 195 345 345 345 F F F 23 460-479 195 195 195 345 345 345 F F F 24 480-499 191 191 191 340 340 340 F F F 25 500-519 195 195 195 345 345 345 F F F 26 520-539 195 195 195 345 345 345 F F F 27 540-559 193 193 193 342 342 342 F F F 28 560-579 193 193 193 342 342 342 F F F 29 580-599 193 193 193 342 342 342 F F F Max gen: 195, Max flush: 345 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.599354776Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:42:08.642209888Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.642603528Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.674333188Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.674971412Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.675013957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.675035539Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.67505343Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974} {"msg":"Crucible 2bc14a46-81aa-4a9e-bf9b-a032211e99b2 has session id: 861e837e-385f-4d33-a941-bbd462e50142","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.675610957Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Upstairs opts: Upstairs UUID: 2bc14a46-81aa-4a9e-bf9b-a032211e99b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.675723978Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Crucible stats registered with UUID: 2bc14a46-81aa-4a9e-bf9b-a032211e99b2","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.675764812Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:42:08.675780004Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676180084Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676205557Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974} {"msg":"2bc14a46-81aa-4a9e-bf9b-a032211e99b2 active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676289565Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676309347Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"0","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.67633543Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"0","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676350381Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"1","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676364673Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"1","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676377264Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"2","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676395076Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"2","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676408557Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.6764326Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676457672Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"ds_connection connected from Ok(127.0.0.1:32968)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676480605Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"ds_connection connected from Ok(127.0.0.1:57758)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676507167Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"ds_connection connected from Ok(127.0.0.1:39404)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.676758353Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.677271684Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"2","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.677301217Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"1","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.677556203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"0","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678067094Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"2","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678095227Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"1","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.67821888Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"0","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678728341Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678765105Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[0]R flush_numbers[0..12]: [345, 345, 339, 340, 341, 344, 345, 345, 345, 345, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678789937Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[0]R generation[0..12]: [195, 195, 190, 191, 192, 194, 195, 195, 195, 195, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678807139Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.67882191Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[1]R flush_numbers[0..12]: [345, 345, 339, 340, 341, 344, 345, 345, 345, 345, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678835912Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[1]R generation[0..12]: [195, 195, 190, 191, 192, 194, 195, 195, 195, 195, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678849413Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678862825Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[2]R flush_numbers[0..12]: [345, 345, 339, 340, 341, 344, 345, 345, 345, 345, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678876256Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[2]R generation[0..12]: [195, 195, 190, 191, 192, 194, 195, 195, 195, 195, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678889747Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678908429Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-02-27T17:42:08.678931852Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678961725Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.678987727Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.67901232Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"":"downstairs","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.679041393Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.679066155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"0","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.679091688Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"1","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.67911736Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"client":"2","":"downstairs","session_id":"861e837e-385f-4d33-a941-bbd462e50142"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.679146583Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.679170956Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"2bc14a46-81aa-4a9e-bf9b-a032211e99b2 is now active with session: 861e837e-385f-4d33-a941-bbd462e50142","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.679190818Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.679203199Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974,"session_id":"861e837e-385f-4d33-a941-bbd462e50142"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:08.67921513Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 350 len 4096 data: 17 02/30 Read block 337 len 24576 03/30 Read block 151 len 4096 04/30 Read block 333 len 4096 05/30 Write block 569 len 32768 data: 12 14 13 17 17 15 16 13 06/30 Read block 114 len 12288 07/30 Flush 08/30 Write block 68 len 4096 data: 16 09/30 Read block 206 len 24576 10/30 Read block 575 len 32768 11/30 Read block 511 len 20480 12/30 Flush 13/30 Write block 568 len 12288 data: 10 13 15 14/30 Read block 61 len 28672 15/30 Read block 163 len 16384 16/30 Write block 501 len 40960 data: 13 15 14 13 15 14 10 10 16 17 17/30 Write block 593 len 28672 data: 13 10 8 7 6 5 3 18/30 Flush 19/30 Write block 123 len 32768 data: 18 18 16 12 11 9 8 9 20/30 Read block 394 len 8192 21/30 Write block 436 len 12288 data: 9 8 11 22/30 Read block 561 len 16384 23/30 Read block 354 len 36864 24/30 Write block 18 len 36864 data: 14 13 12 12 12 11 9 11 11 25/30 Write block 250 len 28672 data: 24 23 23 25 20 16 16 26/30 Read block 202 len 40960 27/30 Write block 244 len 16384 data: 15 18 22 19 28/30 Read block 524 len 16384 29/30 Read block 518 len 36864 30/30 Write block 481 len 12288 data: 9 13 11 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:42:08.700504127Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 345 345 345 T T T 1 020-039 195 195 195 345 345 345 T T T 2 040-059 190 190 190 339 339 339 F F F 3 060-079 196 196 196 347 347 347 F F F 4 080-099 192 192 192 341 341 341 F F F 5 100-119 194 194 194 344 344 344 F F F 6 120-139 195 195 195 345 345 345 T T T 7 140-159 195 195 195 345 345 345 F F F 8 160-179 195 195 195 345 345 345 F F F 9 180-199 195 195 195 345 345 345 F F F 10 200-219 192 192 192 341 341 341 F F F 11 220-239 191 191 191 340 340 340 F F F 12 240-259 195 195 195 345 345 345 T T T 13 260-279 195 195 195 345 345 345 F F F 14 280-299 193 193 193 342 342 342 F F F 15 300-319 191 191 191 340 340 340 F F F 16 320-339 193 193 193 342 342 342 F F F 17 340-359 196 196 196 346 346 346 F F F 18 360-379 191 191 191 340 340 340 F F F 19 380-399 193 193 193 342 342 342 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 193 193 193 342 342 342 T T T 22 440-459 195 195 195 345 345 345 F F F 23 460-479 195 195 195 345 345 345 F F F 24 480-499 191 191 191 340 340 340 T T T 25 500-519 196 196 196 348 348 348 F F F 26 520-539 195 195 195 345 345 345 F F F 27 540-559 193 193 193 342 342 342 F F F 28 560-579 196 196 196 348 348 348 F F F 29 580-599 196 196 196 348 348 348 F F F Max gen: 196, Max flush: 348 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 349 349 349 F F F 1 020-039 197 197 197 349 349 349 F F F 2 040-059 190 190 190 339 339 339 F F F 3 060-079 196 196 196 347 347 347 F F F 4 080-099 192 192 192 341 341 341 F F F 5 100-119 194 194 194 344 344 344 F F F 6 120-139 197 197 197 349 349 349 F F F 7 140-159 195 195 195 345 345 345 F F F 8 160-179 195 195 195 345 345 345 F F F 9 180-199 195 195 195 345 345 345 F F F 10 200-219 192 192 192 341 341 341 F F F 11 220-239 191 191 191 340 340 340 F F F 12 240-259 197 197 197 349 349 349 F F F 13 260-279 195 195 195 345 345 345 F F F 14 280-299 193 193 193 342 342 342 F F F 15 300-319 191 191 191 340 340 340 F F F 16 320-339 193 193 193 342 342 342 F F F 17 340-359 196 196 196 346 346 346 F F F 18 360-379 191 191 191 340 340 340 F F F 19 380-399 193 193 193 342 342 342 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 197 197 197 349 349 349 F F F 22 440-459 195 195 195 345 345 345 F F F 23 460-479 195 195 195 345 345 345 F F F 24 480-499 197 197 197 349 349 349 F F F 25 500-519 196 196 196 348 348 348 F F F 26 520-539 195 195 195 345 345 345 F F F 27 540-559 193 193 193 342 342 342 F F F 28 560-579 196 196 196 348 348 348 F F F 29 580-599 196 196 196 348 348 348 F F F Max gen: 197, Max flush: 349 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:12.98313409Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.029360353Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.030212769Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6e487ab182b38d027fd506ffd486d0e47754015d\",\n git_commit_timestamp: \"2025-02-27T17:06:54.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.030257413Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.030272215Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.030286576Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982} {"msg":"Crucible 12abaff5-8ad7-467b-b7ca-77b27342b99c has session id: 65c18f4f-9502-4b8e-b9ec-2fc092af543c","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.030828021Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Upstairs opts: Upstairs UUID: 12abaff5-8ad7-467b-b7ca-77b27342b99c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.030991867Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Crucible stats registered with UUID: 12abaff5-8ad7-467b-b7ca-77b27342b99c","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.03102302Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:42:13.031038322Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031554994Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031577376Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982} {"msg":"12abaff5-8ad7-467b-b7ca-77b27342b99c active request set","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031652434Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031669836Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"0","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031691458Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"0","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031705369Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"1","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031719521Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"1","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031732942Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"2","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031747204Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"2","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031760525Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031773896Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031786988Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62702)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031917471Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"io task","client":"2","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61148)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031953154Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"io task","client":"1","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51390)","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.031977977Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"io task","client":"0","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.03270618Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"0","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.032750205Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"2","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.033408831Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"2","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.033553466Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"0","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.034001321Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.03419001Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.034822234Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"1","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.035341666Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"1","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"[0]R flush_numbers[0..12]: [349, 349, 339, 347, 341, 344, 349, 345, 345, 345, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.035868389Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"[0]R generation[0..12]: [197, 197, 190, 196, 192, 194, 197, 195, 195, 195, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.035894652Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.035913094Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"[1]R flush_numbers[0..12]: [349, 349, 339, 347, 341, 344, 349, 345, 345, 345, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.035930166Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"[1]R generation[0..12]: [197, 197, 190, 196, 192, 194, 197, 195, 195, 195, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.035947187Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.035964409Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"[2]R flush_numbers[0..12]: [349, 349, 339, 347, 341, 344, 349, 345, 345, 345, 341, 340]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.035981641Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [197, 197, 190, 196, 192, 194, 197, 195, 195, 195, 192, 191]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036010463Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036027995Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036043967Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036060119Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Next flush: 350","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.03607615Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036092082Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"":"downstairs","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036108193Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036123155Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"0","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036139676Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"1","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036155628Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"client":"2","":"downstairs","session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.03617196Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036191002Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"12abaff5-8ad7-467b-b7ca-77b27342b99c is now active with session: 65c18f4f-9502-4b8e-b9ec-2fc092af543c","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036207203Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036221835Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982,"session_id":"65c18f4f-9502-4b8e-b9ec-2fc092af543c"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-02-27T17:42:13.036236936Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 426 len 40960 data: 20 17 16 16 16 18 14 11 11 7 02/30 Read block 229 len 8192 03/30 Read block 18 len 32768 04/30 Write block 567 len 28672 data: 14 11 14 16 14 18 18 05/30 Write block 271 len 40960 data: 10 11 11 13 13 13 14 12 13 11 06/30 Write block 486 len 32768 data: 11 14 17 15 12 13 12 11 07/30 Write block 268 len 40960 data: 15 12 11 11 12 12 14 14 14 15 08/30 Write block 282 len 28672 data: 6 3 3 6 9 13 11 09/30 Write block 359 len 12288 data: 9 13 10 10/30 Read block 151 len 32768 11/30 Read block 74 len 32768 12/30 Read block 282 len 16384 13/30 Write block 527 len 32768 data: 18 17 19 17 18 15 14 12 14/30 Flush 15/30 Write block 355 len 8192 data: 14 15 16/30 Flush 17/30 Write block 137 len 24576 data: 15 13 14 16 17 15 18/30 Read block 236 len 36864 19/30 Write block 482 len 4096 data: 14 20/30 Flush 21/30 Write block 351 len 32768 data: 14 15 13 13 15 16 17 13 22/30 Write block 266 len 8192 data: 12 15 23/30 Read block 253 len 40960 24/30 Flush 25/30 Write block 118 len 40960 data: 18 17 23 24 20 19 19 17 13 12 26/30 Write block 471 len 16384 data: 19 16 13 11 27/30 Write block 564 len 36864 data: 11 14 13 15 12 15 17 15 19 28/30 Write block 197 len 32768 data: 13 10 15 13 13 15 13 12 29/30 Write block 369 len 36864 data: 16 15 18 17 16 14 12 9 11 30/30 Write block 329 len 28672 data: 11 11 11 16 13 18 18 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-02-27T17:42:13.056125162Z","hostname":"w-01JN46VMY2RCGY1N8B5PZCYH9E","pid":1982} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 349 349 349 F F F 1 020-039 197 197 197 349 349 349 F F F 2 040-059 190 190 190 339 339 339 F F F 3 060-079 196 196 196 347 347 347 F F F 4 080-099 192 192 192 341 341 341 F F F 5 100-119 194 194 194 344 344 344 T T T 6 120-139 198 198 198 352 352 352 T T T 7 140-159 198 198 198 352 352 352 F F F 8 160-179 195 195 195 345 345 345 F F F 9 180-199 195 195 195 345 345 345 T T T 10 200-219 192 192 192 341 341 341 T T T 11 220-239 191 191 191 340 340 340 F F F 12 240-259 197 197 197 349 349 349 F F F 13 260-279 198 198 198 353 353 353 F F F 14 280-299 198 198 198 350 350 350 F F F 15 300-319 191 191 191 340 340 340 F F F 16 320-339 193 193 193 342 342 342 T T T 17 340-359 198 198 198 353 353 353 F F F 18 360-379 198 198 198 350 350 350 T T T 19 380-399 193 193 193 342 342 342 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 198 198 198 350 350 350 F F F 22 440-459 195 195 195 345 345 345 F F F 23 460-479 195 195 195 345 345 345 T T T 24 480-499 198 198 198 352 352 352 F F F 25 500-519 196 196 196 348 348 348 F F F 26 520-539 198 198 198 350 350 350 F F F 27 540-559 193 193 193 342 342 342 F F F 28 560-579 198 198 198 350 350 350 T T T 29 580-599 196 196 196 348 348 348 F F F Max gen: 198, Max flush: 353 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 349 349 349 F F F 1 020-039 197 197 197 349 349 349 F F F 2 040-059 190 190 190 339 339 339 F F F 3 060-079 196 196 196 347 347 347 F F F 4 080-099 192 192 192 341 341 341 F F F 5 100-119 199 199 199 354 354 354 F F F 6 120-139 199 199 199 354 354 354 F F F 7 140-159 198 198 198 352 352 352 F F F 8 160-179 195 195 195 345 345 345 F F F 9 180-199 199 199 199 354 354 354 F F F 10 200-219 199 199 199 354 354 354 F F F 11 220-239 191 191 191 340 340 340 F F F 12 240-259 197 197 197 349 349 349 F F F 13 260-279 198 198 198 353 353 353 F F F 14 280-299 198 198 198 350 350 350 F F F 15 300-319 191 191 191 340 340 340 F F F 16 320-339 199 199 199 354 354 354 F F F 17 340-359 198 198 198 353 353 353 F F F 18 360-379 199 199 199 354 354 354 F F F 19 380-399 193 193 193 342 342 342 F F F 20 400-419 182 182 182 326 326 326 F F F 21 420-439 198 198 198 350 350 350 F F F 22 440-459 195 195 195 345 345 345 F F F 23 460-479 199 199 199 354 354 354 F F F 24 480-499 198 198 198 352 352 352 F F F 25 500-519 196 196 196 348 348 348 F F F 26 520-539 198 198 198 350 350 350 F F F 27 540-559 193 193 193 342 342 342 F F F 28 560-579 199 199 199 354 354 354 F F F 29 580-599 196 196 196 348 348 348 F F F Max gen: 199, Max flush: 354 19:24 Test duration Test completed real 19:24.238558629 user 49.444348664 sys 19.677126828 trap 0.064369714 tflt 0.087396715 dflt 0.161809440 kflt 0.006236549 lock 30:34:34.511617184 slp 1:34:35.381652434 lat 5.658826003 stop 0.068050160