Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.2797938 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.021747544 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.019668696 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1186 1187 1188 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.897452298Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.977728737Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.979017915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.979152715Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.979177989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.979196605Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190} {"msg":"Crucible c8e18756-583c-4ae8-b72e-76ab5c639216 has session id: ee7bd751-aeba-4b69-b212-f0ab954efae5","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.980223412Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Upstairs opts: Upstairs UUID: c8e18756-583c-4ae8-b72e-76ab5c639216, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.980322179Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Crucible stats registered with UUID: c8e18756-583c-4ae8-b72e-76ab5c639216","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.980349073Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:18:51.98036284Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981343678Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981419101Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190} {"msg":"c8e18756-583c-4ae8-b72e-76ab5c639216 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981453433Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.9814685Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"0","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981518599Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"0","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981538594Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"1","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981564968Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"1","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981589923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981619086Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981674263Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981701147Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981715944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58660)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981742988Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58084)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981769262Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46775)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.981795646Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.983571144Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"1","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.983599757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.983618393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"0","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.984591393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.984620026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"0","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.984673294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"1","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985522082Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985594256Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985677617Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985696273Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985711459Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985725136Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985738333Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.98575221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985765717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985778794Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.98579594Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985808977Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985821824Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985834371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985935638Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"downstairs","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985954624Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985968411Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"0","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.985990946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"1","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.986011071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.986024898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.986037785Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"c8e18756-583c-4ae8-b72e-76ab5c639216 is now active with session: ee7bd751-aeba-4b69-b212-f0ab954efae5","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.986050972Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.986062679Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:51.986074567Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:18:56.080023954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:18:56.080173011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:18:56.080215551Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:18:56.080247814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.080275968Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"ee7bd751-aeba-4b69-b212-f0ab954efae5"} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.111856045Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:18:56.157376046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.15775479Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.190842516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.191557114Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.191592836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.191606853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.191626039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192} {"msg":"Crucible 3d1c9ed1-e04d-427c-8f06-c985b3f57daf has session id: 52ade1c4-23be-4e3e-bf6a-67637996bfac","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192204768Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Upstairs opts: Upstairs UUID: 3d1c9ed1-e04d-427c-8f06-c985b3f57daf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192260795Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Crucible stats registered with UUID: 3d1c9ed1-e04d-427c-8f06-c985b3f57daf","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192286679Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:18:56.192305775Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192720961Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192755023Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192} {"msg":"3d1c9ed1-e04d-427c-8f06-c985b3f57daf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192808001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192836195Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"0","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192862109Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"0","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192876586Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"1","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192891342Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"1","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192904859Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"2","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192959677Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"2","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.192996728Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.193029301Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.193053116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"ds_connection connected from Ok(127.0.0.1:42547)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.19307875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"io task","client":"0","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"ds_connection connected from Ok(127.0.0.1:63419)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.193106034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"io task","client":"1","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"ds_connection connected from Ok(127.0.0.1:65202)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.193132818Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"io task","client":"2","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.193844626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"2","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.19387207Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"0","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.193886727Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"1","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.194635247Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"1","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.194664971Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"0","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.194692694Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"2","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195361723Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195456511Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195530674Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195556069Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195571245Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195585282Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195598609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195612286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195625713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.19563878Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195652497Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195665054Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195677891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195697297Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195710154Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"":"downstairs","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195722911Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195734798Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"0","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195747845Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"1","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195763022Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"client":"2","":"downstairs","session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195776079Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195787846Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"3d1c9ed1-e04d-427c-8f06-c985b3f57daf is now active with session: 52ade1c4-23be-4e3e-bf6a-67637996bfac","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195800373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195811961Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192,"session_id":"52ade1c4-23be-4e3e-bf6a-67637996bfac"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:18:56.195823758Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 407 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Read block 199 len 32768 03/30 Write block 264 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 388 len 32768 05/30 Write block 513 len 4096 data: 2 06/30 Write block 364 len 32768 data: 2 2 2 2 2 2 2 2 07/30 Read block 61 len 36864 08/30 Flush 09/30 Write block 528 len 16384 data: 2 2 2 2 10/30 Read block 377 len 40960 11/30 Flush 12/30 Write block 272 len 12288 data: 2 2 2 13/30 Read block 148 len 4096 14/30 Flush 15/30 Write block 223 len 24576 data: 2 2 2 2 2 2 16/30 Read block 430 len 20480 17/30 Read block 460 len 36864 18/30 Write block 471 len 8192 data: 2 2 19/30 Read block 329 len 24576 20/30 Flush 21/30 Write block 125 len 12288 data: 2 2 2 22/30 Read block 240 len 32768 23/30 Read block 171 len 24576 24/30 Flush 25/30 Write block 16 len 28672 data: 2 2 2 2 2 2 2 26/30 Read block 362 len 12288 27/30 Write block 320 len 16384 data: 2 2 2 2 28/30 Read block 136 len 8192 29/30 Write block 376 len 20480 data: 2 2 2 2 2 30/30 Read block 502 len 36864 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:18:56.21599367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1192} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 1 1 1 1 1 1 T T T 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 5 5 5 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 2 2 2 T T T 19 380-399 1 1 1 1 1 1 T T T 20 400-419 2 2 2 2 2 2 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 5 5 5 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 6 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 7 7 7 F F F 1 020-039 3 3 3 7 7 7 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 2 2 2 5 5 5 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 3 3 3 7 7 7 F F F 19 380-399 3 3 3 7 7 7 F F F 20 400-419 2 2 2 2 2 2 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 5 5 5 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 2 2 2 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 7 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.492542882Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.538282983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.539010958Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.539061397Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.539076833Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.53909146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200} {"msg":"Crucible 582ffe47-02df-40de-9173-871b147e91b3 has session id: b8500144-dea0-4acd-ba7f-566dd28b673f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.539673618Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Upstairs opts: Upstairs UUID: 582ffe47-02df-40de-9173-871b147e91b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.539717288Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Crucible stats registered with UUID: 582ffe47-02df-40de-9173-871b147e91b3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.539737034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:00.53975306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540346406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.54037323Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200} {"msg":"582ffe47-02df-40de-9173-871b147e91b3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540438185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540456661Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"0","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540480245Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"0","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540494302Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"1","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540508049Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"1","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540525235Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"2","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540539272Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"2","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540552109Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540565556Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540580313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40396)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540690558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33988)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540728949Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43633)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.540774839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.541397558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"2","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.541436009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"0","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.541666057Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"1","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542166184Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"0","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542194757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"1","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542212523Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"2","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542752781Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542782834Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[0]R flush_numbers[0..12]: [7, 7, 1, 1, 1, 1, 6, 1, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542861776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[0]R generation[0..12]: [3, 3, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542881832Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542896308Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[1]R flush_numbers[0..12]: [7, 7, 1, 1, 1, 1, 6, 1, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542909705Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[1]R generation[0..12]: [3, 3, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542922902Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542936529Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[2]R flush_numbers[0..12]: [7, 7, 1, 1, 1, 1, 6, 1, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542949856Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[2]R generation[0..12]: [3, 3, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542963063Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.54297662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.542989537Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543002274Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543014871Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543027439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"":"downstairs","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543043845Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.54306364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"0","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543077027Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"1","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543089785Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"client":"2","":"downstairs","session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543102462Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543114379Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"582ffe47-02df-40de-9173-871b147e91b3 is now active with session: b8500144-dea0-4acd-ba7f-566dd28b673f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543126776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543138463Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200,"session_id":"b8500144-dea0-4acd-ba7f-566dd28b673f"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:00.543150361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 223 len 24576 data: 3 3 3 3 3 3 02/30 Write block 490 len 16384 data: 2 2 2 2 03/30 Write block 260 len 28672 data: 2 2 2 2 3 3 3 04/30 Write block 352 len 28672 data: 2 2 2 2 2 2 2 05/30 Read block 96 len 20480 06/30 Read block 306 len 4096 07/30 Write block 480 len 36864 data: 2 2 2 2 2 2 2 2 2 08/30 Write block 264 len 36864 data: 4 4 4 3 3 3 3 2 3 09/30 Read block 150 len 12288 10/30 Read block 105 len 12288 11/30 Write block 184 len 40960 data: 2 2 2 2 2 2 2 2 2 2 12/30 Read block 531 len 28672 13/30 Read block 60 len 28672 14/30 Read block 528 len 16384 15/30 Write block 401 len 20480 data: 2 2 2 2 2 16/30 Read block 34 len 28672 17/30 Read block 477 len 8192 18/30 Write block 291 len 16384 data: 2 2 2 2 19/30 Write block 81 len 8192 data: 2 2 20/30 Flush 21/30 Write block 144 len 12288 data: 2 2 2 22/30 Read block 69 len 4096 23/30 Write block 82 len 28672 data: 3 2 2 2 2 2 2 24/30 Read block 156 len 28672 25/30 Read block 459 len 20480 26/30 Flush 27/30 Write block 518 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Write block 51 len 12288 data: 2 2 2 29/30 Read block 305 len 40960 30/30 Read block 518 len 24576 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:00.562866346Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1200} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 7 7 7 F F F 1 020-039 3 3 3 7 7 7 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 4 4 4 9 9 9 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 4 4 4 8 8 8 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 4 4 4 8 8 8 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 4 4 4 8 8 8 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 3 3 3 7 7 7 F F F 19 380-399 3 3 3 7 7 7 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 5 5 5 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 2 2 2 2 2 2 T T T 26 520-539 2 2 2 3 3 3 T T T 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 9 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 7 7 7 F F F 1 020-039 3 3 3 7 7 7 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 4 4 4 9 9 9 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 4 4 4 8 8 8 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 4 4 4 8 8 8 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 4 4 4 8 8 8 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 4 4 4 8 8 8 F F F 18 360-379 3 3 3 7 7 7 F F F 19 380-399 3 3 3 7 7 7 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 5 5 5 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 5 5 5 10 10 10 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.790751748Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.836817865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.83754793Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.83759175Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.837607466Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.837622133Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208} {"msg":"Crucible 6a60de92-605d-4c7f-b43c-ca38db401ea0 has session id: 96b1aa83-93db-4fa8-90d3-a7d5ad411d23","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.83820758Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Upstairs opts: Upstairs UUID: 6a60de92-605d-4c7f-b43c-ca38db401ea0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.838259229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Crucible stats registered with UUID: 6a60de92-605d-4c7f-b43c-ca38db401ea0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.838286492Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:04.838309427Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.83882746Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.838850915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208} {"msg":"6a60de92-605d-4c7f-b43c-ca38db401ea0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.83891646Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.838938105Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"0","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.838961Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"0","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.838975436Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"1","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.838990373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"1","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.83900402Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"2","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839018237Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"2","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839031294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839044851Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839058468Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"ds_connection connected from Ok(127.0.0.1:55955)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839151667Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"ds_connection connected from Ok(127.0.0.1:38361)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839186029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"ds_connection connected from Ok(127.0.0.1:61427)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839215162Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839946736Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"2","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.83997512Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"0","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.839991406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"1","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.840653316Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"1","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.84068257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"0","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.840704295Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"2","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841201052Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841319596Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[0]R flush_numbers[0..12]: [7, 7, 10, 1, 9, 1, 6, 9, 1, 8, 1, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841363815Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[0]R generation[0..12]: [3, 3, 5, 1, 4, 1, 2, 4, 1, 4, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841381312Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841395738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[1]R flush_numbers[0..12]: [7, 7, 10, 1, 9, 1, 6, 9, 1, 8, 1, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841409395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[1]R generation[0..12]: [3, 3, 5, 1, 4, 1, 2, 4, 1, 4, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841422932Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841436799Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[2]R flush_numbers[0..12]: [7, 7, 10, 1, 9, 1, 6, 9, 1, 8, 1, 8]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841449946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[2]R generation[0..12]: [3, 3, 5, 1, 4, 1, 2, 4, 1, 4, 1, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841463593Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841487308Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841517631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841539696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841553373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841575778Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0EWait for a query_work_queue command to finish before sending IO ","pid":1208,"":"downstairs","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.84160788Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841631805Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"0","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841651001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"1","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841664518Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"client":"2","":"downstairs","session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841680944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841702929Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"6a60de92-605d-4c7f-b43c-ca38db401ea0 is now active with session: 96b1aa83-93db-4fa8-90d3-a7d5ad411d23","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841727483Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841748248Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208,"session_id":"96b1aa83-93db-4fa8-90d3-a7d5ad411d23"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:04.841761076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 180 len 8192 data: 2 2 02/30 Read block 379 len 36864 03/30 Read block 108 len 24576 04/30 Write block 140 len 36864 data: 2 2 2 2 3 3 3 2 2 05/30 Write block 450 len 40960 data: 2 2 2 2 2 2 2 2 2 2 06/30 Read block 312 len 8192 07/30 Read block 416 len 20480 08/30 Read block 10 len 12288 09/30 Read block 534 len 24576 10/30 Write block 370 len 40960 data: 3 3 2 2 2 2 3 3 3 3 11/30 Read block 402 len 8192 12/30 Read block 496 len 24576 13/30 Read block 550 len 32768 14/30 Flush 15/30 Write block 218 len 28672 data: 2 2 2 2 2 4 4 16/30 Write block 156 len 20480 data: 2 2 2 2 2 17/30 Write block 149 len 36864 data: 2 2 2 2 2 2 2 3 3 18/30 Write block 356 len 12288 data: 3 3 3 19/30 Read block 349 len 8192 20/30 Write block 392 len 28672 data: 2 2 2 2 2 2 2 21/30 Read block 534 len 8192 22/30 Read block 572 len 8192 23/30 Read block 568 len 8192 24/30 Write block 251 len 4096 data: 2 25/30 Write block 116 len 8192 data: 2 2 26/30 Write block 363 len 24576 data: 2 3 3 3 3 3 27/30 Read block 555 len 32768 28/30 Read block 543 len 8192 29/30 Read block 218 len 36864 30/30 Write block 157 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:04.86028501Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1208} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 7 7 7 F F F 1 020-039 3 3 3 7 7 7 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 2 2 2 6 6 6 F F F 7 140-159 6 6 6 11 11 11 T T T 8 160-179 1 1 1 1 1 1 T T T 9 180-199 6 6 6 11 11 11 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 4 4 4 8 8 8 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 4 4 4 8 8 8 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 4 4 4 8 8 8 T T T 18 360-379 6 6 6 11 11 11 T T T 19 380-399 3 3 3 7 7 7 T T T 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 2 2 2 5 5 5 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 5 5 5 10 10 10 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 11 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 3 3 3 7 7 7 F F F 1 020-039 3 3 3 7 7 7 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 7 7 7 12 12 12 F F F 6 120-139 2 2 2 6 6 6 F F F 7 140-159 7 7 7 12 12 12 F F F 8 160-179 7 7 7 12 12 12 F F F 9 180-199 6 6 6 11 11 11 F F F 10 200-219 7 7 7 12 12 12 F F F 11 220-239 7 7 7 12 12 12 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 4 4 4 8 8 8 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 7 7 7 12 12 12 F F F 19 380-399 7 7 7 12 12 12 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 2 2 2 5 5 5 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 5 5 5 10 10 10 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 7, Max flush: 12 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.124289444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.170451619Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.171245499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.171301546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.171319452Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.171334279Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216} {"msg":"Crucible adcd27de-c516-4ba4-9cbe-82d34594941d has session id: 7717485e-c418-4ab3-9af8-b681daa0d5da","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172005397Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Upstairs opts: Upstairs UUID: adcd27de-c516-4ba4-9cbe-82d34594941d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172039319Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Crucible stats registered with UUID: adcd27de-c516-4ba4-9cbe-82d34594941d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172058935Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:09.172072032Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172543385Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172615249Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216} {"msg":"adcd27de-c516-4ba4-9cbe-82d34594941d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172634314Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172648271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"0","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172669836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"0","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172683413Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"1","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.17269748Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"1","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172710727Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"2","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172724564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"2","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172746219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172771793Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172797817Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"ds_connection connected from Ok(127.0.0.1:55401)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.172846706Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"ds_connection connected from Ok(127.0.0.1:34792)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.173103218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"ds_connection connected from Ok(127.0.0.1:52498)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.17313774Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.173708171Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"0","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.173736565Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"1","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.173752551Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"2","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.174320293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"0","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.174428948Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"1","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.174468029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"2","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.174865959Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.17494866Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[0]R flush_numbers[0..12]: [7, 7, 10, 1, 9, 12, 6, 12, 12, 11, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175056266Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[0]R generation[0..12]: [3, 3, 5, 1, 4, 7, 2, 7, 7, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175077041Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175091748Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[1]R flush_numbers[0..12]: [7, 7, 10, 1, 9, 12, 6, 12, 12, 11, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175105615Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[1]R generation[0..12]: [3, 3, 5, 1, 4, 7, 2, 7, 7, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.17512376Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175139147Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[2]R flush_numbers[0..12]: [7, 7, 10, 1, 9, 12, 6, 12, 12, 11, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175152754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[2]R generation[0..12]: [3, 3, 5, 1, 4, 7, 2, 7, 7, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175166041Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175179568Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175192005Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175204872Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175217369Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175229836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"":"downstairs","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"No downstairs reconciliation required","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175251102Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175273377Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"0","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175298591Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"1","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175324905Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"client":"2","":"downstairs","session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175350299Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175374304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"adcd27de-c516-4ba4-9cbe-82d34594941d is now active with session: 7717485e-c418-4ab3-9af8-b681daa0d5da","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175404257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175428962Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216,"session_id":"7717485e-c418-4ab3-9af8-b681daa0d5da"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:09.175453436Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 216 len 32768 data: 2 2 3 3 3 3 3 5 03/30 Flush 04/30 Write block 386 len 20480 data: 2 2 2 2 2 05/30 Read block 99 len 20480 06/30 Read block 586 len 28672 07/30 Read block 190 len 16384 08/30 Write block 12 len 40960 data: 2 2 2 2 3 3 3 3 3 3 09/30 Write block 251 len 24576 data: 3 2 2 2 2 2 10/30 Flush 11/30 Write block 486 len 24576 data: 3 3 3 2 3 3 12/30 Write block 364 len 8192 data: 4 4 13/30 Read block 428 len 8192 14/30 Write block 124 len 28672 data: 2 3 3 3 2 2 2 15/30 Write block 577 len 32768 data: 2 2 2 2 2 2 2 2 16/30 Read block 219 len 8192 17/30 Read block 76 len 20480 18/30 Read block 545 len 24576 19/30 Read block 293 len 4096 20/30 Write block 207 len 12288 data: 2 2 2 21/30 Flush 22/30 Write block 302 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Read block 370 len 4096 24/30 Read block 179 len 8192 25/30 Flush 26/30 Write block 231 len 24576 data: 2 2 2 2 2 2 27/30 Read block 407 len 32768 28/30 Read block 120 len 4096 29/30 Write block 522 len 16384 data: 3 3 3 3 30/30 Write block 115 len 16384 data: 2 3 3 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:09.194228194Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1216} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 7 7 7 12 12 12 T T T 6 120-139 8 8 8 16 16 16 F F F 7 140-159 7 7 7 12 12 12 F F F 8 160-179 7 7 7 12 12 12 F F F 9 180-199 6 6 6 11 11 11 F F F 10 200-219 8 8 8 16 16 16 F F F 11 220-239 8 8 8 14 14 14 T T T 12 240-259 8 8 8 15 15 15 F F F 13 260-279 4 4 4 8 8 8 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 8 8 8 17 17 17 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 2 2 2 5 5 5 F F F 24 480-499 8 8 8 16 16 16 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 5 5 5 10 10 10 T T T 27 540-559 1 1 1 1 1 1 F F F 28 560-579 8 8 8 16 16 16 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 8, Max flush: 17 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 9 9 9 F F F 5 100-119 9 9 9 18 18 18 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 7 7 7 12 12 12 F F F 8 160-179 7 7 7 12 12 12 F F F 9 180-199 6 6 6 11 11 11 F F F 10 200-219 8 8 8 16 16 16 F F F 11 220-239 9 9 9 18 18 18 F F F 12 240-259 8 8 8 15 15 15 F F F 13 260-279 4 4 4 8 8 8 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 8 8 8 17 17 17 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 2 2 2 5 5 5 F F F 24 480-499 8 8 8 16 16 16 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 9 9 9 18 18 18 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 8 8 8 16 16 16 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 9, Max flush: 18 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.42514573Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.470622031Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.471315724Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.471356855Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.47137577Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.471392496Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224} {"msg":"Crucible 9b320582-3ef3-4fc6-a5df-941b51fccd21 has session id: 54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472079221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Upstairs opts: Upstairs UUID: 9b320582-3ef3-4fc6-a5df-941b51fccd21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472118062Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Crucible stats registered with UUID: 9b320582-3ef3-4fc6-a5df-941b51fccd21","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472134148Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:13.472147476Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472679645Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472704859Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224} {"msg":"9b320582-3ef3-4fc6-a5df-941b51fccd21 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472765545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472783272Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"0","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472805766Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"0","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472819513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"1","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.47283364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"1","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472846907Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"2","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472861534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"2","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.47287829Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472903804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472927669Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"ds_connection connected from Ok(127.0.0.1:51064)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.472942535Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"ds_connection connected from Ok(127.0.0.1:33738)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.473041493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"ds_connection connected from Ok(127.0.0.1:46622)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.473205536Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.473738565Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"0","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.473767569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"1","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.473782905Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"2","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.474397186Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"0","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.474426219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"2","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.474447225Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"1","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.474983883Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475094088Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[0]R flush_numbers[0..12]: [15, 15, 10, 1, 9, 18, 16, 12, 12, 11, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475143177Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[0]R generation[0..12]: [8, 8, 5, 1, 4, 9, 8, 7, 7, 6, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475165542Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475180698Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[1]R flush_numbers[0..12]: [15, 15, 10, 1, 9, 18, 16, 12, 12, 11, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475194805Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[1]R generation[0..12]: [8, 8, 5, 1, 4, 9, 8, 7, 7, 6, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475208472Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475222229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[2]R flush_numbers[0..12]: [15, 15, 10, 1, 9, 18, 16, 12, 12, 11, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475235516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[2]R generation[0..12]: [8, 8, 5, 1, 4, 9, 8, 7, 7, 6, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475248513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.47526193Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475274687Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2025-01-15T05:19:13.475287784Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475314558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475338653Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"":"downstairs","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475361108Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475382663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"0","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475400689Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"1","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475413876Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"client":"2","":"downstairs","session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475427662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.47543954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"9b320582-3ef3-4fc6-a5df-941b51fccd21 is now active with session: 54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475452117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475463814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224,"session_id":"54f3fd6a-e5fb-4abd-ba1c-fca65c2caa06"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:13.475475812Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 307 len 20480 data: 3 3 3 2 2 04/30 Flush 05/30 Write block 488 len 8192 data: 4 3 06/30 Write block 558 len 20480 data: 2 2 2 2 2 07/30 Read block 554 len 16384 08/30 Read block 467 len 24576 09/30 Read block 15 len 8192 10/30 Write block 189 len 16384 data: 3 3 3 3 11/30 Write block 141 len 40960 data: 3 3 3 4 4 4 3 3 3 3 12/30 Write block 474 len 12288 data: 2 2 2 13/30 Read block 436 len 16384 14/30 Read block 585 len 4096 15/30 Read block 543 len 16384 16/30 Read block 210 len 36864 17/30 Write block 500 len 8192 data: 2 2 18/30 Read block 401 len 4096 19/30 Read block 273 len 28672 20/30 Read block 535 len 12288 21/30 Read block 433 len 12288 22/30 Read block 23 len 40960 23/30 Read block 565 len 28672 24/30 Write block 274 len 4096 data: 3 25/30 Read block 485 len 16384 26/30 Read block 90 len 24576 27/30 Write block 154 len 40960 data: 3 3 4 5 3 3 3 2 2 2 28/30 Write block 77 len 32768 data: 2 2 2 2 3 4 3 3 29/30 Read block 504 len 8192 30/30 Read block 156 len 32768 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Write 5 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:13.494128167Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1224} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 1 1 1 1 1 1 T T T 4 080-099 4 4 4 9 9 9 T T T 5 100-119 9 9 9 18 18 18 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 7 7 7 12 12 12 T T T 8 160-179 7 7 7 12 12 12 T T T 9 180-199 6 6 6 11 11 11 T T T 10 200-219 8 8 8 16 16 16 F F F 11 220-239 9 9 9 18 18 18 F F F 12 240-259 8 8 8 15 15 15 F F F 13 260-279 4 4 4 8 8 8 T T T 14 280-299 4 4 4 8 8 8 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 2 2 2 5 5 5 T T T 24 480-499 8 8 8 16 16 16 T T T 25 500-519 5 5 5 10 10 10 T T T 26 520-539 9 9 9 18 18 18 F F F 27 540-559 1 1 1 1 1 1 T T T 28 560-579 8 8 8 16 16 16 T T T 29 580-599 8 8 8 16 16 16 F F F Max gen: 10, Max flush: 21 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 11 11 11 22 22 22 F F F 4 080-099 11 11 11 22 22 22 F F F 5 100-119 9 9 9 18 18 18 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 11 11 11 22 22 22 F F F 8 160-179 11 11 11 22 22 22 F F F 9 180-199 11 11 11 22 22 22 F F F 10 200-219 8 8 8 16 16 16 F F F 11 220-239 9 9 9 18 18 18 F F F 12 240-259 8 8 8 15 15 15 F F F 13 260-279 11 11 11 22 22 22 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 11 11 11 F F F 23 460-479 11 11 11 22 22 22 F F F 24 480-499 11 11 11 22 22 22 F F F 25 500-519 11 11 11 22 22 22 F F F 26 520-539 9 9 9 18 18 18 F F F 27 540-559 11 11 11 22 22 22 F F F 28 560-579 11 11 11 22 22 22 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 11, Max flush: 22 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.768099454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.812989557Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.813677951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.813721121Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.813735788Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.813750315Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232} {"msg":"Crucible 33bdefa0-6555-4d68-8688-6811bd647a99 has session id: bafe114f-c9a6-43b0-b536-bb1315ad6750","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.814440549Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Upstairs opts: Upstairs UUID: 33bdefa0-6555-4d68-8688-6811bd647a99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.814488448Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Crucible stats registered with UUID: 33bdefa0-6555-4d68-8688-6811bd647a99","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.814513042Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:17.814533577Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.814989094Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815021567Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232} {"msg":"33bdefa0-6555-4d68-8688-6811bd647a99 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815088242Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815106288Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"0","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815129752Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"0","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815144399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"1","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815163985Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"1","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815188519Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"2","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815217093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"2","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815248435Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.81527468Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815302623Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"ds_connection connected from Ok(127.0.0.1:36065)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815480673Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"ds_connection connected from Ok(127.0.0.1:46354)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815518364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"ds_connection connected from Ok(127.0.0.1:44481)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815543469Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.815993307Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"0","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.816040956Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"1","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.816170846Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"2","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.816692918Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"1","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.816722362Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"2","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.816739828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"0","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817254731Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817280185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[0]R flush_numbers[0..12]: [15, 15, 10, 22, 22, 18, 16, 22, 22, 22, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817352109Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[0]R generation[0..12]: [8, 8, 5, 11, 11, 9, 8, 11, 11, 11, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817369175Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817383572Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[1]R flush_numbers[0..12]: [15, 15, 10, 22, 22, 18, 16, 22, 22, 22, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817396889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[1]R generation[0..12]: [8, 8, 5, 11, 11, 9, 8, 11, 11, 11, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817410206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817423993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[2]R flush_numbers[0..12]: [15, 15, 10, 22, 22, 18, 16, 22, 22, 22, 16, 18]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.81743727Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[2]R generation[0..12]: [8, 8, 5, 11, 11, 9, 8, 11, 11, 11, 8, 9]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817450487Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817475221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:19:17.817499276Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817530709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817556643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817573179Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"":"downstairs","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817586426Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817601562Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"0","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817625297Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"1","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817651931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"client":"2","":"downstairs","session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817667388Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817687273Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"33bdefa0-6555-4d68-8688-6811bd647a99 is now active with session: bafe114f-c9a6-43b0-b536-bb1315ad6750","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817710598Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817733503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232,"session_id":"bafe114f-c9a6-43b0-b536-bb1315ad6750"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:17.817754418Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 539 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 424 len 16384 05/30 Read block 575 len 20480 06/30 Read block 524 len 28672 07/30 Read block 402 len 16384 08/30 Read block 305 len 4096 09/30 Read block 313 len 4096 10/30 Flush 11/30 Write block 106 len 12288 data: 2 2 2 12/30 Read block 365 len 36864 13/30 Write block 495 len 32768 data: 2 2 2 2 2 3 3 2 14/30 Read block 217 len 16384 15/30 Read block 269 len 24576 16/30 Write block 560 len 4096 data: 3 17/30 Read block 10 len 12288 18/30 Read block 93 len 28672 19/30 Read block 586 len 8192 20/30 Read block 245 len 36864 21/30 Read block 480 len 12288 22/30 Read block 320 len 36864 23/30 Flush 24/30 Write block 235 len 24576 data: 3 3 2 2 2 2 25/30 Read block 230 len 16384 26/30 Write block 448 len 32768 data: 2 2 3 3 3 3 3 3 27/30 Write block 235 len 32768 data: 4 4 3 3 3 3 2 2 28/30 Read block 34 len 24576 29/30 Write block 237 len 36864 data: 4 4 4 4 3 3 2 2 2 30/30 Write block 20 len 36864 data: 4 4 3 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:17.837597294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1232} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 8 8 8 15 15 15 T T T 2 040-059 5 5 5 10 10 10 F F F 3 060-079 11 11 11 22 22 22 F F F 4 080-099 11 11 11 22 22 22 F F F 5 100-119 12 12 12 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 11 11 11 22 22 22 F F F 8 160-179 11 11 11 22 22 22 F F F 9 180-199 11 11 11 22 22 22 F F F 10 200-219 8 8 8 16 16 16 F F F 11 220-239 9 9 9 18 18 18 T T T 12 240-259 8 8 8 15 15 15 T T T 13 260-279 11 11 11 22 22 22 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 11 11 11 T T T 23 460-479 11 11 11 22 22 22 F F F 24 480-499 12 12 12 26 26 26 F F F 25 500-519 12 12 12 26 26 26 F F F 26 520-539 12 12 12 25 25 25 F F F 27 540-559 12 12 12 25 25 25 F F F 28 560-579 12 12 12 26 26 26 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 12, Max flush: 26 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 13 13 13 27 27 27 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 11 11 11 22 22 22 F F F 4 080-099 11 11 11 22 22 22 F F F 5 100-119 12 12 12 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 11 11 11 22 22 22 F F F 8 160-179 11 11 11 22 22 22 F F F 9 180-199 11 11 11 22 22 22 F F F 10 200-219 8 8 8 16 16 16 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 13 13 13 27 27 27 F F F 13 260-279 11 11 11 22 22 22 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 13 13 13 27 27 27 F F F 23 460-479 11 11 11 22 22 22 F F F 24 480-499 12 12 12 26 26 26 F F F 25 500-519 12 12 12 26 26 26 F F F 26 520-539 12 12 12 25 25 25 F F F 27 540-559 12 12 12 25 25 25 F F F 28 560-579 12 12 12 26 26 26 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 13, Max flush: 27 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.071602671Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.117754698Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.118628311Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.11867331Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.118689597Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.118704354Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240} {"msg":"Crucible e0208d04-cc63-40f4-a620-d77a6c5eb227 has session id: 7b13dc20-3ea9-42ad-a5d2-6829ed715b84","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.11924842Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Upstairs opts: Upstairs UUID: e0208d04-cc63-40f4-a620-d77a6c5eb227, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.11929227Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Crucible stats registered with UUID: e0208d04-cc63-40f4-a620-d77a6c5eb227","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119319534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:22.119334531Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119766973Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119790977Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240} {"msg":"e0208d04-cc63-40f4-a620-d77a6c5eb227 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119838897Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119865031Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"0","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119889285Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"0","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119903712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"1","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119918539Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"1","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119931925Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"2","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119945822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"2","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119959089Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119972526Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.119985823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"ds_connection connected from Ok(127.0.0.1:46154)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.1201338Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"ds_connection connected from Ok(127.0.0.1:50770)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.1201789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"ds_connection connected from Ok(127.0.0.1:44194)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.120205523Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.120858955Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"2","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.120887619Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"0","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.120905915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"1","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.121444913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"2","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.121484604Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"1","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.121598408Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"0","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122062173Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122196683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[0]R flush_numbers[0..12]: [15, 27, 10, 22, 22, 26, 16, 22, 22, 22, 16, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122267716Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[0]R generation[0..12]: [8, 13, 5, 11, 11, 12, 8, 11, 11, 11, 8, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.12229555Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122310947Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[1]R flush_numbers[0..12]: [15, 27, 10, 22, 22, 26, 16, 22, 22, 22, 16, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122324943Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[1]R generation[0..12]: [8, 13, 5, 11, 11, 12, 8, 11, 11, 11, 8, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.12234223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122356256Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[2]R flush_numbers[0..12]: [15, 27, 10, 22, 22, 26, 16, 22, 22, 22, 16, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122369553Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[2]R generation[0..12]: [8, 13, 5, 11, 11, 12, 8, 11, 11, 11, 8, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.12238275Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122396317Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg"Wait for a query_work_queue command to finish before sending IO :"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122416863Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.12243029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122443037Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122455604Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"":"downstairs","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122468291Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122480078Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"0","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122493035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"1","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122505822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"client":"2","":"downstairs","session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.12251859Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122530507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"e0208d04-cc63-40f4-a620-d77a6c5eb227 is now active with session: 7b13dc20-3ea9-42ad-a5d2-6829ed715b84","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122542884Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122556251Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240,"session_id":"7b13dc20-3ea9-42ad-a5d2-6829ed715b84"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:22.122568428Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 83 len 8192 data: 4 4 02/30 Read block 557 len 36864 03/30 Write block 60 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 418 len 4096 05/30 Read block 570 len 16384 06/30 Write block 522 len 24576 data: 4 4 4 4 3 2 07/30 Write block 179 len 20480 data: 2 3 3 2 2 08/30 Read block 248 len 28672 09/30 Read block 457 len 32768 10/30 Write block 89 len 16384 data: 2 2 2 2 11/30 Read block 60 len 4096 12/30 Read block 221 len 24576 13/30 Read block 204 len 32768 14/30 Read block 74 len 28672 15/30 Read block 263 len 12288 16/30 Write block 192 len 36864 data: 4 3 2 2 2 2 2 2 2 17/30 Write block 173 len 8192 data: 2 2 18/30 Flush 19/30 Flush 20/30 Write block 248 len 16384 data: 2 2 2 4 21/30 Write block 447 len 28672 data: 2 3 3 4 4 4 4 22/30 Read block 360 len 8192 23/30 Write block 429 len 28672 data: 2 2 2 2 2 2 2 24/30 Read block 370 len 32768 25/30 Write block 512 len 36864 data: 2 3 2 2 2 2 3 3 3 26/30 Read block 236 len 28672 27/30 Write block 519 len 8192 data: 4 4 28/30 Write block 536 len 8192 data: 2 2 29/30 Write block 278 len 36864 data: 2 2 2 2 2 2 2 2 2 30/30 Read block 451 len 32768 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 11 10 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:22.142590954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1240} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 13 13 13 27 27 27 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 14 14 14 28 28 28 F F F 4 080-099 14 14 14 28 28 28 F F F 5 100-119 12 12 12 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 11 11 11 22 22 22 F F F 8 160-179 14 14 14 28 28 28 F F F 9 180-199 14 14 14 28 28 28 F F F 10 200-219 14 14 14 28 28 28 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 13 13 13 27 27 27 T T T 13 260-279 11 11 11 22 22 22 T T T 14 280-299 4 4 4 8 8 8 T T T 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 1 1 1 1 1 1 T T T 22 440-459 13 13 13 27 27 27 T T T 23 460-479 11 11 11 22 22 22 F F F 24 480-499 12 12 12 26 26 26 F F F 25 500-519 12 12 12 26 26 26 T T T 26 520-539 14 14 14 28 28 28 T T T 27 540-559 12 12 12 25 25 25 F F F 28 560-579 12 12 12 26 26 26 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 14, Max flush: 28 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 F F F 1 020-039 13 13 13 27 27 27 F F F 2 040-059 5 5 5 10 10 10 F F F 3 060-079 14 14 14 28 28 28 F F F 4 080-099 14 14 14 28 28 28 F F F 5 100-119 12 12 12 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 11 11 11 22 22 22 F F F 8 160-179 14 14 14 28 28 28 F F F 9 180-199 14 14 14 28 28 28 F F F 10 200-219 14 14 14 28 28 28 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 15 15 15 29 29 29 F F F 14 280-299 15 15 15 29 29 29 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 15 15 15 29 29 29 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 11 11 11 22 22 22 F F F 24 480-499 12 12 12 26 26 26 F F F 25 500-519 15 15 15 29 29 29 F F F 26 520-539 15 15 15 29 29 29 F F F 27 540-559 12 12 12 25 25 25 F F F 28 560-579 12 12 12 26 26 26 F F F 29 580-599 8 8 8 16 16 16 F F F Max gen: 15, Max flush: 29 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.393076199Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.438560278Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.4392164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.439265298Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.439288223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.439306679Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248} {"msg":"Crucible 519dd7fe-92fc-45b5-a149-a2babc6c7404 has session id: 22c5b5d6-ea9f-44fa-bbb0-90a9699feabd","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.439851446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Upstairs opts: Upstairs UUID: 519dd7fe-92fc-45b5-a149-a2babc6c7404, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.43991962Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Crucible stats registered with UUID: 519dd7fe-92fc-45b5-a149-a2babc6c7404","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.439949334Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:26.439974878Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440430165Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440460058Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248} {"msg":"519dd7fe-92fc-45b5-a149-a2babc6c7404 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440488231Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440503968Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"0","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440530712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"0","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440552117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"1","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.44058231Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"1","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440610734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"2","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440641757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"2","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440679638Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440708631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440735305Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:52391)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440762209Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:62179)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440786524Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"ds_connection connected from Ok(127.0.0.1:48899)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.440914115Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.441540033Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"0","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.441573995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"2","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.442137208Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"0","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.442285954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"2","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.442573769Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"1","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.442749789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.442840479Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443095271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"1","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[0]R flush_numbers[0..12]: [15, 27, 10, 28, 28, 26, 16, 22, 28, 28, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443571033Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[0]R generation[0..12]: [8, 13, 5, 14, 14, 12, 8, 11, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443596037Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443614004Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[1]R flush_numbers[0..12]: [15, 27, 10, 28, 28, 26, 16, 22, 28, 28, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443631289Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[1]R generation[0..12]: [8, 13, 5, 14, 14, 12, 8, 11, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443647976Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443665002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[2]R flush_numbers[0..12]: [15, 27, 10, 28, 28, 26, 16, 22, 28, 28, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443681428Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[2]R generation[0..12]: [8, 13, 5, 14, 14, 12, 8, 11, 14, 14, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443697734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-15T05:19:26.443714691Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443735626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443751452Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443767139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443782725Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"":"downstairs","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443798822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443818047Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"0","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443834783Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"1","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.44385077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"client":"2","":"downstairs","session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443880293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.44389624Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"519dd7fe-92fc-45b5-a149-a2babc6c7404 is now active with session: 22c5b5d6-ea9f-44fa-bbb0-90a9699feabd","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443912026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2025-01-15T05:19:26.443927093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248,"session_id":"22c5b5d6-ea9f-44fa-bbb0-90a9699feabd"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:26.443946568Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 80 len 24576 data: 3 4 5 5 5 3 02/30 Read block 423 len 40960 03/30 Write block 581 len 24576 data: 3 3 3 3 2 2 04/30 Read block 532 len 16384 05/30 Write block 194 len 16384 data: 3 3 3 3 06/30 Read block 325 len 28672 07/30 Flush 08/30 Write block 487 len 8192 data: 4 5 09/30 Write block 175 len 32768 data: 2 2 2 2 3 4 4 3 10/30 Write block 37 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Flush 12/30 Flush 13/30 Write block 519 len 32768 data: 5 5 3 5 5 5 5 4 14/30 Read block 569 len 8192 15/30 Read block 60 len 12288 16/30 Read block 400 len 36864 17/30 Read block 461 len 28672 18/30 Flush 19/30 Write block 157 len 40960 data: 6 4 4 4 3 3 3 2 2 2 20/30 Read block 155 len 24576 21/30 Write block 579 len 24576 data: 3 3 4 4 4 4 22/30 Read block 153 len 12288 23/30 Read block 158 len 4096 24/30 Write block 1 len 16384 data: 2 2 2 2 25/30 Read block 38 len 28672 26/30 Write block 263 len 8192 data: 3 5 27/30 Write block 157 len 28672 data: 7 5 5 5 4 4 4 28/30 Read block 435 len 40960 29/30 Read block 422 len 24576 30/30 Write block 38 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:26.464259508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1248} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 15 15 15 T T T 1 020-039 16 16 16 31 31 31 T T T 2 040-059 16 16 16 31 31 31 T T T 3 060-079 14 14 14 28 28 28 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 12 12 12 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 11 11 11 22 22 22 T T T 8 160-179 16 16 16 31 31 31 T T T 9 180-199 16 16 16 31 31 31 F F F 10 200-219 14 14 14 28 28 28 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 15 15 15 29 29 29 T T T 14 280-299 15 15 15 29 29 29 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 15 15 15 29 29 29 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 11 11 11 22 22 22 F F F 24 480-499 16 16 16 31 31 31 F F F 25 500-519 16 16 16 33 33 33 F F F 26 520-539 16 16 16 33 33 33 F F F 27 540-559 12 12 12 25 25 25 F F F 28 560-579 12 12 12 26 26 26 T T T 29 580-599 16 16 16 30 30 30 T T T Max gen: 16, Max flush: 33 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 34 34 34 F F F 1 020-039 17 17 17 34 34 34 F F F 2 040-059 17 17 17 34 34 34 F F F 3 060-079 14 14 14 28 28 28 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 12 12 12 26 26 26 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 17 17 17 34 34 34 F F F 8 160-179 17 17 17 34 34 34 F F F 9 180-199 16 16 16 31 31 31 F F F 10 200-219 14 14 14 28 28 28 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 17 17 17 34 34 34 F F F 14 280-299 15 15 15 29 29 29 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 15 15 15 29 29 29 F F F 22 440-459 15 15 15 29 29 29 F F F 23 460-479 11 11 11 22 22 22 F F F 24 480-499 16 16 16 31 31 31 F F F 25 500-519 16 16 16 33 33 33 F F F 26 520-539 16 16 16 33 33 33 F F F 27 540-559 12 12 12 25 25 25 F F F 28 560-579 17 17 17 34 34 34 F F F 29 580-599 17 17 17 34 34 34 F F F Max gen: 17, Max flush: 34 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.720463956Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.765349711Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.766106029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.76614822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.766164206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.766179043Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256} {"msg":"Crucible b095d8cc-2726-437f-aec8-c20a9cd7a55b has session id: 9c672a84-4320-4949-b765-69c8b7ad24e2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.766819508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Upstairs opts: Upstairs UUID: b095d8cc-2726-437f-aec8-c20a9cd7a55b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.766870686Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Crucible stats registered with UUID: b095d8cc-2726-437f-aec8-c20a9cd7a55b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.76689577Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:30.766913906Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767388059Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767411873Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256} {"msg":"b095d8cc-2726-437f-aec8-c20a9cd7a55b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767467411Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767487027Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"0","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767523868Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"0","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767550722Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"1","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767580075Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"1","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767610399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"2","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767641432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"2","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767669885Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767694509Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767721263Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47283)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767747748Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40716)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767773891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62766)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.767798526Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.768494019Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"0","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.768522602Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"2","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.769133503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"0","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.769302935Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"2","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.769749664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.769876995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.770137476Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"1","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.770617327Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"1","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[0]R flush_numbers[0..12]: [34, 34, 34, 28, 30, 26, 16, 34, 34, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771129502Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[0]R generation[0..12]: [17, 17, 17, 14, 16, 12, 8, 17, 17, 16, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771151386Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771167283Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[1]R flush_numbers[0..12]: [34, 34, 34, 28, 30, 26, 16, 34, 34, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.77118128Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[1]R generation[0..12]: [17, 17, 17, 14, 16, 12, 8, 17, 17, 16, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771196047Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771209993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[2]R flush_numbers[0..12]: [34, 34, 34, 28, 30, 26, 16, 34, 34, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.77122347Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[2]R generation[0..12]: [17, 17, 17, 14, 16, 12, 8, 17, 17, 16, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771237117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771250984Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.77126841Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:19:30.771282717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771299203Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.77131201Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"":"downstairs","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771325007Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771336875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"0","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771350951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"1","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771364038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"client":"2","":"downstairs","session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771377126Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771389353Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"b095d8cc-2726-437f-aec8-c20a9cd7a55b is now active with session: 9c672a84-4320-4949-b765-69c8b7ad24e2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.77140211Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771414307Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256,"session_id":"9c672a84-4320-4949-b765-69c8b7ad24e2"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:30.771426284Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 583 len 36864 data: 5 5 3 3 2 2 2 2 2 02/30 Read block 456 len 40960 03/30 Read block 535 len 36864 04/30 Write block 514 len 24576 data: 3 3 3 3 4 6 05/30 Write block 536 len 28672 data: 3 3 2 3 3 3 3 06/30 Read block 21 len 40960 07/30 Read block 185 len 8192 08/30 Read block 129 len 4096 09/30 Read block 45 len 4096 10/30 Flush 11/30 Write block 1 len 36864 data: 3 3 3 3 2 2 2 2 2 12/30 Read block 107 len 28672 13/30 Write block 56 len 28672 data: 2 2 2 2 3 3 3 14/30 Read block 122 len 36864 15/30 Read block 5 len 24576 16/30 Write block 425 len 40960 data: 2 2 2 2 3 3 3 3 3 3 17/30 Write block 160 len 32768 data: 6 5 5 5 3 3 3 2 18/30 Read block 440 len 24576 19/30 Read block 103 len 16384 20/30 Write block 111 len 24576 data: 2 2 2 2 3 4 21/30 Read block 63 len 20480 22/30 Write block 441 len 16384 data: 2 2 2 2 23/30 Write block 265 len 20480 data: 5 5 4 4 4 24/30 Write block 569 len 36864 data: 2 2 2 2 2 2 2 2 3 25/30 Read block 260 len 16384 26/30 Read block 540 len 20480 27/30 Write block 433 len 8192 data: 4 4 28/30 Read block 553 len 4096 29/30 Write block 159 len 24576 data: 6 7 6 6 6 4 30/30 Write block 268 len 24576 data: 5 5 4 3 4 3 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 9 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:30.791675718Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1256} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 34 34 34 T T T 1 020-039 17 17 17 34 34 34 F F F 2 040-059 17 17 17 34 34 34 T T T 3 060-079 14 14 14 28 28 28 T T T 4 080-099 16 16 16 30 30 30 F F F 5 100-119 12 12 12 26 26 26 T T T 6 120-139 8 8 8 16 16 16 F F F 7 140-159 17 17 17 34 34 34 T T T 8 160-179 17 17 17 34 34 34 T T T 9 180-199 16 16 16 31 31 31 F F F 10 200-219 14 14 14 28 28 28 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 17 17 17 34 34 34 T T T 14 280-299 15 15 15 29 29 29 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 15 15 15 29 29 29 T T T 22 440-459 15 15 15 29 29 29 T T T 23 460-479 11 11 11 22 22 22 F F F 24 480-499 16 16 16 31 31 31 F F F 25 500-519 18 18 18 35 35 35 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 17 17 17 34 34 34 T T T 29 580-599 18 18 18 35 35 35 F F F Max gen: 18, Max flush: 35 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 36 36 36 F F F 1 020-039 17 17 17 34 34 34 F F F 2 040-059 19 19 19 36 36 36 F F F 3 060-079 19 19 19 36 36 36 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 19 19 19 36 36 36 F F F 6 120-139 8 8 8 16 16 16 F F F 7 140-159 19 19 19 36 36 36 F F F 8 160-179 19 19 19 36 36 36 F F F 9 180-199 16 16 16 31 31 31 F F F 10 200-219 14 14 14 28 28 28 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 19 19 19 36 36 36 F F F 14 280-299 15 15 15 29 29 29 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 3 3 3 7 7 7 F F F 17 340-359 7 7 7 12 12 12 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 19 19 19 36 36 36 F F F 22 440-459 19 19 19 36 36 36 F F F 23 460-479 11 11 11 22 22 22 F F F 24 480-499 16 16 16 31 31 31 F F F 25 500-519 18 18 18 35 35 35 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 18 18 18 35 35 35 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 18 18 18 35 35 35 F F F Max gen: 19, Max flush: 36 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.057581401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.102811697Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.103660824Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.103706584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.10372521Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.103743755Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264} {"msg":"Crucible ffd22e94-6fa2-4762-a2b4-1b7b553e1507 has session id: fa175131-9656-447a-9e8f-2be5f9b222cf","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.104419233Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Upstairs opts: Upstairs UUID: ffd22e94-6fa2-4762-a2b4-1b7b553e1507, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.104468282Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Crucible stats registered with UUID: ffd22e94-6fa2-4762-a2b4-1b7b553e1507","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.104486567Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:35.104500324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.104987594Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105017877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264} {"msg":"ffd22e94-6fa2-4762-a2b4-1b7b553e1507 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105079903Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105107737Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"0","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105144928Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"0","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105174002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"1","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105204875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"1","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105231439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"2","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105255043Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"2","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105277268Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105305902Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105374576Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:37803)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105422865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:38645)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105467095Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:54783)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.105702612Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.106141603Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"2","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.106171096Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"0","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.106248688Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"1","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.106862509Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"1","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.106891073Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"2","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.106907069Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"0","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107436709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107462403Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[0]R flush_numbers[0..12]: [36, 34, 36, 36, 30, 36, 16, 36, 36, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107480719Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[0]R generation[0..12]: [19, 17, 19, 19, 16, 19, 8, 19, 19, 16, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107495456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107509613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[1]R flush_numbers[0..12]: [36, 34, 36, 36, 30, 36, 16, 36, 36, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107527049Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[1]R generation[0..12]: [19, 17, 19, 19, 16, 19, 8, 19, 19, 16, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107540826Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107554772Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[2]R flush_numbers[0..12]: [36, 34, 36, 36, 30, 36, 16, 36, 36, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.10756808Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[2]R generation[0..12]: [19, 17, 19, 19, 16, 19, 8, 19, 19, 16, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107581426Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107594903Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.10760748Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107620288Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107632905Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107645722Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"":"downstairs","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107658639Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107670246Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"0","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107690442Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"1","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107703409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"client":"2","":"downstairs","session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107718445Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107730643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"ffd22e94-6fa2-4762-a2b4-1b7b553e1507 is now active with session: fa175131-9656-447a-9e8f-2be5f9b222cf","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.10774327Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107755007Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264,"session_id":"fa175131-9656-447a-9e8f-2be5f9b222cf"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:35.107766875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 345 len 36864 data: 2 2 2 2 2 2 2 3 3 02/30 Read block 246 len 32768 03/30 Read block 593 len 8192 04/30 Write block 121 len 28672 data: 2 2 2 3 4 4 4 05/30 Read block 91 len 36864 06/30 Read block 530 len 4096 07/30 Read block 57 len 4096 08/30 Read block 66 len 20480 09/30 Read block 82 len 20480 10/30 Flush 11/30 Write block 101 len 32768 data: 2 2 2 2 2 3 3 3 12/30 Read block 17 len 20480 13/30 Write block 171 len 24576 data: 2 2 3 3 3 3 14/30 Write block 338 len 16384 data: 2 2 2 2 15/30 Read block 424 len 24576 16/30 Flush 17/30 Write block 543 len 36864 data: 3 3 3 2 2 2 2 2 2 18/30 Read block 1 len 20480 19/30 Read block 377 len 4096 20/30 Read block 237 len 40960 21/30 Write block 166 len 8192 data: 4 3 22/30 Read block 56 len 24576 23/30 Write block 157 len 12288 data: 8 6 7 24/30 Read block 345 len 28672 25/30 Flush 26/30 Flush 27/30 Write block 186 len 36864 data: 3 3 3 4 4 4 5 4 4 28/30 Read block 101 len 32768 29/30 Read block 7 len 40960 30/30 Write block 172 len 40960 data: 3 4 4 4 4 3 3 4 5 5 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:35.12853892Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1264} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 36 36 36 F F F 1 020-039 17 17 17 34 34 34 F F F 2 040-059 19 19 19 36 36 36 F F F 3 060-079 19 19 19 36 36 36 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 20 20 20 38 38 38 F F F 6 120-139 20 20 20 37 37 37 F F F 7 140-159 20 20 20 39 39 39 F F F 8 160-179 20 20 20 39 39 39 T T T 9 180-199 16 16 16 31 31 31 T T T 10 200-219 14 14 14 28 28 28 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 19 19 19 36 36 36 F F F 14 280-299 15 15 15 29 29 29 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 19 19 19 36 36 36 F F F 22 440-459 19 19 19 36 36 36 F F F 23 460-479 11 11 11 22 22 22 F F F 24 480-499 16 16 16 31 31 31 F F F 25 500-519 18 18 18 35 35 35 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 20 20 20 39 39 39 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 18 18 18 35 35 35 F F F Max gen: 20, Max flush: 39 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 36 36 36 F F F 1 020-039 17 17 17 34 34 34 F F F 2 040-059 19 19 19 36 36 36 F F F 3 060-079 19 19 19 36 36 36 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 20 20 20 38 38 38 F F F 6 120-139 20 20 20 37 37 37 F F F 7 140-159 20 20 20 39 39 39 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 21 21 21 40 40 40 F F F 10 200-219 14 14 14 28 28 28 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 15 15 15 29 29 29 F F F 13 260-279 19 19 19 36 36 36 F F F 14 280-299 15 15 15 29 29 29 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 20 20 20 38 38 38 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 F F F 20 400-419 4 4 4 8 8 8 F F F 21 420-439 19 19 19 36 36 36 F F F 22 440-459 19 19 19 36 36 36 F F F 23 460-479 11 11 11 22 22 22 F F F 24 480-499 16 16 16 31 31 31 F F F 25 500-519 18 18 18 35 35 35 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 20 20 20 39 39 39 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 18 18 18 35 35 35 F F F Max gen: 21, Max flush: 40 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.351158095Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.397254515Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.397924823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.397978921Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.398003625Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.398022441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272} {"msg":"Crucible 1757bd6c-b75e-45e9-9b14-58814befd353 has session id: 5ca43347-ce81-4b73-96a9-6b2151d0a028","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.398763953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Upstairs opts: Upstairs UUID: 1757bd6c-b75e-45e9-9b14-58814befd353, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.398801905Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Crucible stats registered with UUID: 1757bd6c-b75e-45e9-9b14-58814befd353","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.398816481Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:39.398829198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399201884Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399224449Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272} {"msg":"1757bd6c-b75e-45e9-9b14-58814befd353 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399280586Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.39930702Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"0","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399331055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"0","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399345712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"1","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399360368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"1","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399433682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"2","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399470583Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"2","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399498997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399526261Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399552375Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"ds_connection connected from Ok(127.0.0.1:53502)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399578079Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"ds_connection connected from Ok(127.0.0.1:42588)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399604993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"ds_connection connected from Ok(127.0.0.1:52136)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.399674717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.40032564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"1","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.400356373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"2","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.400371789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"0","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.40102955Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"0","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401054774Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"1","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401070261Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"2","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401615998Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401638772Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[0]R flush_numbers[0..12]: [36, 34, 36, 36, 30, 38, 37, 39, 40, 40, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401670095Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[0]R generation[0..12]: [19, 17, 19, 19, 16, 20, 20, 20, 21, 21, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401684802Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401698789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[1]R flush_numbers[0..12]: [36, 34, 36, 36, 30, 38, 37, 39, 40, 40, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401712325Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[1]R generation[0..12]: [19, 17, 19, 19, 16, 20, 20, 20, 21, 21, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401725602Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401740309Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[2]R flush_numbers[0..12]: [36, 34, 36, 36, 30, 38, 37, 39, 40, 40, 28, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401753696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[2]R generation[0..12]: [19, 17, 19, 19, 16, 20, 20, 20, 21, 21, 14, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401767033Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.40178064Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401793347Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401806634Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401819441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401832158Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"":"downstairs","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401845046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401865311Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"0","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401896484Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"1","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401922358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"client":"2","":"downstairs","session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401945683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.4019591Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"1757bd6c-b75e-45e9-9b14-58814befd353 is now active with session: 5ca43347-ce81-4b73-96a9-6b2151d0a028","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401973166Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.401993912Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272,"session_id":"5ca43347-ce81-4b73-96a9-6b2151d0a028"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:39.402017016Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 355 len 12288 data: 3 4 4 02/30 Read block 386 len 20480 03/30 Write block 77 len 32768 data: 3 3 3 4 5 6 6 6 04/30 Read block 222 len 20480 05/30 Read block 272 len 36864 06/30 Read block 264 len 8192 07/30 Write block 281 len 8192 data: 3 3 08/30 Read block 527 len 40960 09/30 Read block 245 len 40960 10/30 Read block 308 len 28672 11/30 Read block 229 len 40960 12/30 Read block 505 len 36864 13/30 Write block 97 len 8192 data: 2 2 14/30 Write block 508 len 32768 data: 2 2 2 2 3 4 4 4 15/30 Read block 295 len 8192 16/30 Write block 136 len 12288 data: 2 2 2 17/30 Write block 155 len 20480 data: 4 5 9 7 8 18/30 Write block 242 len 12288 data: 4 3 3 19/30 Read block 339 len 24576 20/30 Write block 487 len 36864 data: 5 6 4 4 4 3 3 2 3 21/30 Write block 199 len 20480 data: 3 3 2 2 2 22/30 Write block 507 len 40960 data: 2 3 3 3 3 4 5 5 5 4 23/30 Read block 361 len 32768 24/30 Write block 38 len 8192 data: 4 4 25/30 Write block 468 len 24576 data: 2 2 2 3 3 2 26/30 Read block 507 len 12288 27/30 Read block 87 len 20480 28/30 Write block 396 len 40960 data: 3 3 3 2 2 3 3 3 3 3 29/30 Read block 158 len 20480 30/30 Read block 270 len 40960 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 30 29 30 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:39.422227739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1272} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 36 36 36 F F F 1 020-039 17 17 17 34 34 34 T T T 2 040-059 19 19 19 36 36 36 F F F 3 060-079 19 19 19 36 36 36 T T T 4 080-099 16 16 16 30 30 30 T T T 5 100-119 20 20 20 38 38 38 F F F 6 120-139 20 20 20 37 37 37 T T T 7 140-159 20 20 20 39 39 39 T T T 8 160-179 21 21 21 40 40 40 F F F 9 180-199 21 21 21 40 40 40 T T T 10 200-219 14 14 14 28 28 28 T T T 11 220-239 13 13 13 27 27 27 F F F 12 240-259 15 15 15 29 29 29 T T T 13 260-279 19 19 19 36 36 36 F F F 14 280-299 15 15 15 29 29 29 T T T 15 300-319 10 10 10 21 21 21 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 20 20 20 38 38 38 T T T 18 360-379 8 8 8 16 16 16 F F F 19 380-399 8 8 8 15 15 15 T T T 20 400-419 4 4 4 8 8 8 T T T 21 420-439 19 19 19 36 36 36 F F F 22 440-459 19 19 19 36 36 36 F F F 23 460-479 11 11 11 22 22 22 T T T 24 480-499 16 16 16 31 31 31 T T T 25 500-519 18 18 18 35 35 35 T T T 26 520-539 18 18 18 35 35 35 F F F 27 540-559 20 20 20 39 39 39 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 18 18 18 35 35 35 F F F Max gen: 21, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 36 36 36 F F F 1 020-039 22 22 22 41 41 41 F F F 2 040-059 19 19 19 36 36 36 F F F 3 060-079 22 22 22 41 41 41 F F F 4 080-099 22 22 22 41 41 41 F F F 5 100-119 20 20 20 38 38 38 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 22 22 22 41 41 41 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 22 22 22 41 41 41 F F F 11 220-239 13 13 13 27 27 27 F F F 12 240-259 22 22 22 41 41 41 F F F 13 260-279 19 19 19 36 36 36 F F F 14 280-299 22 22 22 41 41 41 F F F 15 300-319 10 10 10 21 21 21 F F F 16 320-339 20 20 20 38 38 38 F F F 17 340-359 22 22 22 41 41 41 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 22 22 22 41 41 41 F F F 20 400-419 22 22 22 41 41 41 F F F 21 420-439 19 19 19 36 36 36 F F F 22 440-459 19 19 19 36 36 36 F F F 23 460-479 22 22 22 41 41 41 F F F 24 480-499 22 22 22 41 41 41 F F F 25 500-519 22 22 22 41 41 41 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 20 20 20 39 39 39 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 18 18 18 35 35 35 F F F Max gen: 22, Max flush: 41 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.714651285Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.760890872Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.761605461Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.76165016Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.761666287Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.761680893Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280} {"msg":"Crucible 6660eaf0-1f0b-4e1f-b634-3c597ab9da7c has session id: e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.762372537Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Upstairs opts: Upstairs UUID: 6660eaf0-1f0b-4e1f-b634-3c597ab9da7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.762417377Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Crucible stats registered with UUID: 6660eaf0-1f0b-4e1f-b634-3c597ab9da7c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.762441092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:43.762467286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.762881722Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.762908886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280} {"msg":"6660eaf0-1f0b-4e1f-b634-3c597ab9da7c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.762971761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.762990097Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"0","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763014482Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"0","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763030398Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"1","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763055432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"1","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763080936Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"2","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.76310915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"2","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763135854Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763163178Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763192541Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:64687)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763218775Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:39979)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.76324244Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40883)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.76341754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763944821Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"1","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.763987721Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"0","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.764112163Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"2","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.764670007Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"0","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.76469849Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"2","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.764714726Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"1","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765261733Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765296655Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[0]R flush_numbers[0..12]: [36, 41, 36, 41, 41, 38, 41, 41, 40, 41, 41, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765323419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[0]R generation[0..12]: [19, 22, 19, 22, 22, 20, 22, 22, 21, 22, 22, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765339175Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765353622Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[1]R flush_numbers[0..12]: [36, 41, 36, 41, 41, 38, 41, 41, 40, 41, 41, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765367689Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[1]R generation[0..12]: [19, 22, 19, 22, 22, 20, 22, 22, 21, 22, 22, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765381066Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765398192Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[2]R flush_numbers[0..12]: [36, 41, 36, 41, 41, 38, 41, 41, 40, 41, 41, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765412048Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[2]R generation[0..12]: [19, 22, 19, 22, 22, 20, 22, 22, 21, 22, 22, 13]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765425516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765439033Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Max found gen is 23","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765451659Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Generation requested: 24 >= found:23","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765464437Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765476904Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765489351Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"":"downstairs","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765502178Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:19:43.765513836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"0","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765531801Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"1","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765544598Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"client":"2","":"downstairs","session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765557376Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765569393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"6660eaf0-1f0b-4e1f-b634-3c597ab9da7c is now active with session: e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.76558193Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765596027Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280,"session_id":"e5535a9e-6ee3-4aba-bc1e-9e6e55bd61e1"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:43.765608054Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 463 len 8192 data: 2 2 02/30 Write block 497 len 24576 data: 3 3 3 4 4 3 03/30 Read block 200 len 4096 04/30 Read block 523 len 4096 05/30 Read block 231 len 32768 06/30 Read block 281 len 12288 07/30 Read block 234 len 20480 08/30 Write block 40 len 24576 data: 4 3 3 3 3 2 09/30 Write block 238 len 4096 data: 5 10/30 Flush 11/30 Write block 418 len 36864 data: 2 2 2 2 2 2 2 3 3 12/30 Read block 1 len 32768 13/30 Write block 321 len 12288 data: 3 3 3 14/30 Flush 15/30 Write block 212 len 28672 data: 2 2 2 2 3 3 4 16/30 Read block 504 len 24576 17/30 Read block 339 len 40960 18/30 Read block 434 len 4096 19/30 Read block 458 len 32768 20/30 Read block 141 len 24576 21/30 Write block 302 len 16384 data: 3 3 3 3 22/30 Read block 450 len 12288 23/30 Flush 24/30 Write block 450 len 32768 data: 5 5 5 5 4 4 3 3 25/30 Write block 506 len 4096 data: 2 26/30 Write block 289 len 8192 data: 2 2 27/30 Write block 321 len 20480 data: 4 4 4 2 2 28/30 Read block 403 len 8192 29/30 Write block 108 len 24576 data: 4 2 2 3 3 3 30/30 Write block 333 len 32768 data: 2 2 2 2 2 3 3 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 6 Sent Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:43.783536033Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1280} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 36 36 36 F F F 1 020-039 22 22 22 41 41 41 F F F 2 040-059 24 24 24 42 42 42 F F F 3 060-079 22 22 22 41 41 41 F F F 4 080-099 22 22 22 41 41 41 F F F 5 100-119 20 20 20 38 38 38 T T T 6 120-139 22 22 22 41 41 41 F F F 7 140-159 22 22 22 41 41 41 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 24 24 24 44 44 44 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 22 22 22 41 41 41 F F F 13 260-279 19 19 19 36 36 36 F F F 14 280-299 22 22 22 41 41 41 T T T 15 300-319 24 24 24 44 44 44 F F F 16 320-339 24 24 24 43 43 43 T T T 17 340-359 22 22 22 41 41 41 T T T 18 360-379 8 8 8 16 16 16 F F F 19 380-399 22 22 22 41 41 41 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 24 24 24 43 43 43 F F F 22 440-459 19 19 19 36 36 36 T T T 23 460-479 24 24 24 42 42 42 F F F 24 480-499 24 24 24 42 42 42 F F F 25 500-519 24 24 24 42 42 42 T T T 26 520-539 18 18 18 35 35 35 F F F 27 540-559 20 20 20 39 39 39 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 18 18 18 35 35 35 F F F Max gen: 24, Max flush: 44 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 36 36 36 F F F 1 020-039 22 22 22 41 41 41 F F F 2 040-059 24 24 24 42 42 42 F F F 3 060-079 22 22 22 41 41 41 F F F 4 080-099 22 22 22 41 41 41 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 22 22 22 41 41 41 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 22 22 22 41 41 41 F F F 10 200-219 24 24 24 44 44 44 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 22 22 22 41 41 41 F F F 13 260-279 19 19 19 36 36 36 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 24 24 24 44 44 44 F F F 16 320-339 25 25 25 45 45 45 F F F 17 340-359 25 25 25 45 45 45 F F F 18 360-379 8 8 8 16 16 16 F F F 19 380-399 22 22 22 41 41 41 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 24 24 24 43 43 43 F F F 22 440-459 25 25 25 45 45 45 F F F 23 460-479 24 24 24 42 42 42 F F F 24 480-499 24 24 24 42 42 42 F F F 25 500-519 25 25 25 45 45 45 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 20 20 20 39 39 39 F F F 28 560-579 19 19 19 36 36 36 F F F 29 580-599 18 18 18 35 35 35 F F F Max gen: 25, Max flush: 45 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.038633414Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:48.082956546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.083352346Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.115711867Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.116303033Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.116337585Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.116355821Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.116370318Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288} {"msg":"Crucible 973752ed-65ef-49c3-a7c7-1c2586706f53 has session id: af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117042686Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Upstairs opts: Upstairs UUID: 973752ed-65ef-49c3-a7c7-1c2586706f53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117076078Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Crucible stats registered with UUID: 973752ed-65ef-49c3-a7c7-1c2586706f53","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117090665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:48.117103352Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117493224Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117527046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288} {"msg":"973752ed-65ef-49c3-a7c7-1c2586706f53 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117583563Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117616226Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"0","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117652208Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"0","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117683461Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"1","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117716393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"1","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117748206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"2","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117781689Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"2","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117820849Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.117859021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.11803782Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"ds_connection connected from Ok(127.0.0.1:63942)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.118078111Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"ds_connection connected from Ok(127.0.0.1:41332)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.118138258Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"ds_connection connected from Ok(127.0.0.1:51215)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.118156253Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.118598373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"2","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.118643163Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"1","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.118677006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"0","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.119264702Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"2","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.119404101Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"1","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.119439293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"0","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.119784495Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120059143Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[0]R flush_numbers[0..12]: [36, 41, 42, 41, 41, 45, 41, 41, 40, 41, 44, 42]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120087886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[0]R generation[0..12]: [19, 22, 24, 22, 22, 25, 22, 22, 21, 22, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120104812Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120119279Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[1]R flush_numbers[0..12]: [36, 41, 42, 41, 41, 45, 41, 41, 40, 41, 44, 42]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120132766Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[1]R generation[0..12]: [19, 22, 24, 22, 22, 25, 22, 22, 21, 22, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120146153Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.12015977Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[2]R flush_numbers[0..12]: [36, 41, 42, 41, 41, 45, 41, 41, 40, 41, 44, 42]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120173287Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[2]R generation[0..12]: [19, 22, 24, 22, 22, 25, 22, 22, 21, 22, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120186364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120199931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120212528Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120225365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120237892Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:19:48.120250499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"":"downstairs","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120269405Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120283252Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"0","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120296589Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"1","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120309356Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"client":"2","":"downstairs","session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120322083Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.12033395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"973752ed-65ef-49c3-a7c7-1c2586706f53 is now active with session: af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120346688Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120358215Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288,"session_id":"af5a2ad5-b1d9-418a-9c2f-f9fcdb6e4761"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:48.120370022Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 3 len 28672 data: 4 4 3 3 3 3 3 02/30 Write block 576 len 40960 data: 3 4 3 4 4 5 5 6 6 4 03/30 Write block 360 len 24576 data: 2 2 2 3 5 5 04/30 Read block 396 len 28672 05/30 Flush 06/30 Write block 573 len 8192 data: 3 3 07/30 Read block 130 len 20480 08/30 Write block 185 len 40960 data: 3 4 4 4 5 5 5 6 5 5 09/30 Read block 548 len 16384 10/30 Read block 178 len 4096 11/30 Read block 206 len 32768 12/30 Write block 154 len 12288 data: 4 5 6 13/30 Write block 81 len 16384 data: 6 7 7 7 14/30 Read block 530 len 8192 15/30 Write block 481 len 20480 data: 3 3 3 3 3 16/30 Read block 374 len 36864 17/30 Flush 18/30 Write block 378 len 24576 data: 4 4 3 2 2 2 19/30 Read block 55 len 28672 20/30 Write block 259 len 12288 data: 2 3 3 21/30 Write block 191 len 4096 data: 6 22/30 Write block 211 len 40960 data: 2 3 3 3 3 4 4 5 4 4 23/30 Write block 550 len 24576 data: 3 3 2 2 2 2 24/30 Write block 348 len 24576 data: 3 3 3 3 4 4 25/30 Read block 343 len 24576 26/30 Write block 65 len 40960 data: 3 3 2 2 2 2 2 2 2 2 27/30 Read block 349 len 8192 28/30 Read block 461 len 40960 29/30 Write block 323 len 32768 data: 5 3 3 2 2 2 2 2 30/30 Read block 270 len 40960 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:48.140463781Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1288} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 22 22 22 41 41 41 F F F 2 040-059 24 24 24 42 42 42 F F F 3 060-079 22 22 22 41 41 41 T T T 4 080-099 26 26 26 47 47 47 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 26 26 26 47 47 47 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 26 26 26 47 47 47 T T T 10 200-219 24 24 24 44 44 44 T T T 11 220-239 24 24 24 42 42 42 T T T 12 240-259 22 22 22 41 41 41 T T T 13 260-279 19 19 19 36 36 36 T T T 14 280-299 25 25 25 45 45 45 F F F 15 300-319 24 24 24 44 44 44 F F F 16 320-339 25 25 25 45 45 45 T T T 17 340-359 25 25 25 45 45 45 T T T 18 360-379 26 26 26 46 46 46 T T T 19 380-399 22 22 22 41 41 41 T T T 20 400-419 24 24 24 43 43 43 F F F 21 420-439 24 24 24 43 43 43 F F F 22 440-459 25 25 25 45 45 45 F F F 23 460-479 24 24 24 42 42 42 F F F 24 480-499 26 26 26 47 47 47 F F F 25 500-519 25 25 25 45 45 45 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 20 20 20 39 39 39 T T T 28 560-579 26 26 26 47 47 47 F F F 29 580-599 26 26 26 46 46 46 F F F Max gen: 26, Max flush: 47 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 22 22 22 41 41 41 F F F 2 040-059 24 24 24 42 42 42 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 26 26 26 47 47 47 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 26 26 26 47 47 47 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 27 27 27 48 48 48 F F F 10 200-219 27 27 27 48 48 48 F F F 11 220-239 27 27 27 48 48 48 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 27 27 27 48 48 48 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 24 24 24 44 44 44 F F F 16 320-339 27 27 27 48 48 48 F F F 17 340-359 27 27 27 48 48 48 F F F 18 360-379 27 27 27 48 48 48 F F F 19 380-399 27 27 27 48 48 48 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 24 24 24 43 43 43 F F F 22 440-459 25 25 25 45 45 45 F F F 23 460-479 24 24 24 42 42 42 F F F 24 480-499 26 26 26 47 47 47 F F F 25 500-519 25 25 25 45 45 45 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 27 27 27 48 48 48 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 26 26 26 46 46 46 F F F Max gen: 27, Max flush: 48 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.451033919Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.496026969Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.496653527Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.496696337Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.496711354Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.496725751Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296} {"msg":"Crucible f07d3634-5a7e-4c4e-a4a7-92ec6e9bf16a has session id: fbc90942-1030-499d-8944-a137218ef749","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497261809Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Upstairs opts: Upstairs UUID: f07d3634-5a7e-4c4e-a4a7-92ec6e9bf16a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497322796Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Crucible stats registered with UUID: f07d3634-5a7e-4c4e-a4a7-92ec6e9bf16a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.49734777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:52.497368495Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.49779188Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497815914Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296} {"msg":"f07d3634-5a7e-4c4e-a4a7-92ec6e9bf16a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497881949Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497903214Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"0","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497927969Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"0","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497948034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"1","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497972868Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"1","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.497998573Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"2","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498026706Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"2","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498051531Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498078944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498104649Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"ds_connection connected from Ok(127.0.0.1:41738)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498130063Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"ds_connection connected from Ok(127.0.0.1:44832)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498170124Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"ds_connection connected from Ok(127.0.0.1:45105)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498202776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498859028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"0","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.498890271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"1","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.499517419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"1","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.499651048Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"0","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.49977747Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"2","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.500121662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.500226298Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.500518302Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"2","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[0]R flush_numbers[0..12]: [46, 41, 42, 48, 47, 45, 41, 47, 40, 48, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501046103Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[0]R generation[0..12]: [26, 22, 24, 27, 26, 25, 22, 26, 21, 27, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501069827Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501085294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[1]R flush_numbers[0..12]: [46, 41, 42, 48, 47, 45, 41, 47, 40, 48, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.5010995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[1]R generation[0..12]: [26, 22, 24, 27, 26, 25, 22, 26, 21, 27, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501113627Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501127644Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[2]R flush_numbers[0..12]: [46, 41, 42, 48, 47, 45, 41, 47, 40, 48, 48, 48]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501141181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[2]R generation[0..12]: [26, 22, 24, 27, 26, 25, 22, 26, 21, 27, 27, 27]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501154588Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501172814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501186011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:19:52.501198908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501218204Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501241198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"":"downstairs","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501263093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501275771Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"0","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501288737Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"1","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501309913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"client":"2","":"downstairs","session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.50132332Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501335547Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"f07d3634-5a7e-4c4e-a4a7-92ec6e9bf16a is now active with session: fbc90942-1030-499d-8944-a137218ef749","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501347964Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501359601Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296,"session_id":"fbc90942-1030-499d-8944-a137218ef749"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:52.501371789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 303 len 8192 data: 4 4 02/30 Read block 324 len 16384 03/30 Read block 527 len 32768 04/30 Write block 198 len 20480 data: 3 4 4 3 3 05/30 Flush 06/30 Write block 420 len 40960 data: 3 3 3 3 3 4 4 3 3 4 07/30 Read block 149 len 16384 08/30 Read block 99 len 20480 09/30 Write block 389 len 40960 data: 3 3 2 3 3 3 3 4 4 4 10/30 Write block 273 len 12288 data: 4 4 2 11/30 Write block 45 len 32768 data: 3 2 2 2 2 2 3 3 12/30 Read block 167 len 24576 13/30 Read block 315 len 40960 14/30 Read block 29 len 20480 15/30 Read block 171 len 32768 16/30 Write block 236 len 4096 data: 5 17/30 Write block 349 len 16384 data: 4 4 4 5 18/30 Read block 182 len 8192 19/30 Write block 221 len 28672 data: 4 4 6 5 4 4 4 20/30 Read block 91 len 8192 21/30 Read block 439 len 28672 22/30 Write block 582 len 36864 data: 6 7 7 5 4 3 3 3 3 23/30 Read block 31 len 4096 24/30 Flush 25/30 Write block 190 len 28672 data: 6 7 7 6 6 4 4 26/30 Read block 588 len 12288 27/30 Read block 264 len 20480 28/30 Read block 179 len 12288 29/30 Write block 26 len 36864 data: 3 3 3 2 2 2 2 2 2 30/30 Write block 234 len 20480 data: 3 5 6 5 6 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:52.521464958Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1296} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 22 22 22 41 41 41 T T T 2 040-059 28 28 28 50 50 50 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 26 26 26 47 47 47 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 26 26 26 47 47 47 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 28 28 28 49 49 49 T T T 10 200-219 28 28 28 49 49 49 F F F 11 220-239 28 28 28 50 50 50 T T T 12 240-259 27 27 27 48 48 48 F F F 13 260-279 28 28 28 50 50 50 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 28 28 28 49 49 49 F F F 16 320-339 27 27 27 48 48 48 F F F 17 340-359 28 28 28 50 50 50 F F F 18 360-379 27 27 27 48 48 48 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 25 25 25 45 45 45 F F F 23 460-479 24 24 24 42 42 42 F F F 24 480-499 26 26 26 47 47 47 F F F 25 500-519 25 25 25 45 45 45 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 27 27 27 48 48 48 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 28, Max flush: 50 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 29 29 29 51 51 51 F F F 2 040-059 28 28 28 50 50 50 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 26 26 26 47 47 47 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 26 26 26 47 47 47 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 29 29 29 51 51 51 F F F 10 200-219 28 28 28 49 49 49 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 28 28 28 50 50 50 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 28 28 28 49 49 49 F F F 16 320-339 27 27 27 48 48 48 F F F 17 340-359 28 28 28 50 50 50 F F F 18 360-379 27 27 27 48 48 48 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 28 28 28 50 50 50 F F F 22 440-459 25 25 25 45 45 45 F F F 23 460-479 24 24 24 42 42 42 F F F 24 480-499 26 26 26 47 47 47 F F F 25 500-519 25 25 25 45 45 45 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 27 27 27 48 48 48 F F F 28 560-579 26 26 26 47 47 47 F F F 29 580-599 28 28 28 50 50 50 F F F Max gen: 29, Max flush: 51 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.757977178Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:56.801488313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.801895321Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.834107675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.834851837Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.834895297Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.834912783Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.834931059Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304} {"msg":"Crucible 3b74eee5-9899-424f-a3b5-032192623a52 has session id: 06d19af2-78c2-40a0-988d-390450356e4d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.835635929Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Upstairs opts: Upstairs UUID: 3b74eee5-9899-424f-a3b5-032192623a52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.8356756Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Crucible stats registered with UUID: 3b74eee5-9899-424f-a3b5-032192623a52","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.835692187Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:56.835705054Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836133807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836157951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304} {"msg":"3b74eee5-9899-424f-a3b5-032192623a52 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836233954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.8362522Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"0","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836277434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"0","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836304358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"1","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836333821Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"1","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836365084Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"2","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836395867Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"2","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836424301Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836453334Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836523678Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55896)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836644331Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49506)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836679943Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39545)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.836704418Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.83731165Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"0","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.837339923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"2","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.83735507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"1","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.837882431Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"1","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.837971211Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"0","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.837997814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"2","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838391415Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838534643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[0]R flush_numbers[0..12]: [46, 51, 50, 48, 47, 45, 41, 47, 40, 51, 49, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838645878Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[0]R generation[0..12]: [26, 29, 28, 27, 26, 25, 22, 26, 21, 29, 28, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838665183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.83867985Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[1]R flush_numbers[0..12]: [46, 51, 50, 48, 47, 45, 41, 47, 40, 51, 49, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838697946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[1]R generation[0..12]: [26, 29, 28, 27, 26, 25, 22, 26, 21, 29, 28, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838711683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.83872559Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[2]R flush_numbers[0..12]: [46, 51, 50, 48, 47, 45, 41, 47, 40, 51, 49, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838738997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[2]R generation[0..12]: [26, 29, 28, 27, 26, 25, 22, 26, 21, 29, 28, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838752244Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838765651Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838778178Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838791035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838803702Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":"downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838816189Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"":Wait for a query_work_queue command to finish before sending IO "downstairs","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838835475Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838847562Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"0","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838860429Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"1","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838873156Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"client":"2","":"downstairs","session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838885883Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.83890004Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"3b74eee5-9899-424f-a3b5-032192623a52 is now active with session: 06d19af2-78c2-40a0-988d-390450356e4d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838912787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838924485Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304,"session_id":"06d19af2-78c2-40a0-988d-390450356e4d"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:19:56.838936352Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 567 len 12288 data: 2 2 3 02/30 Write block 353 len 36864 data: 5 3 4 5 5 4 2 3 3 03/30 Write block 513 len 28672 data: 6 6 6 5 4 5 7 04/30 Write block 312 len 40960 data: 2 2 2 2 2 2 2 2 3 5 05/30 Write block 140 len 40960 data: 3 4 4 4 5 5 5 4 4 4 06/30 Write block 195 len 32768 data: 5 5 4 4 5 5 4 4 07/30 Read block 320 len 16384 08/30 Write block 205 len 20480 data: 2 2 3 3 3 09/30 Flush 10/30 Write block 458 len 28672 data: 3 3 2 2 2 3 3 11/30 Read block 213 len 16384 12/30 Write block 263 len 24576 data: 4 6 6 6 5 6 13/30 Read block 12 len 40960 14/30 Write block 516 len 8192 data: 6 5 15/30 Write block 551 len 8192 data: 4 3 16/30 Read block 455 len 16384 17/30 Read block 505 len 40960 18/30 Read block 339 len 40960 19/30 Read block 309 len 36864 20/30 Write block 585 len 40960 data: 6 5 4 4 4 4 3 2 2 2 21/30 Write block 463 len 40960 data: 4 4 2 2 2 3 3 3 4 4 22/30 Write block 145 len 28672 data: 6 6 5 5 5 4 3 23/30 Write block 434 len 4096 data: 5 24/30 Read block 339 len 24576 25/30 Read block 562 len 36864 26/30 Read block 446 len 12288 27/30 Read block 117 len 32768 28/30 Read block 380 len 24576 29/30 Write block 87 len 32768 data: 3 3 3 3 3 3 2 2 30/30 Read block 549 len 16384 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 7 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 20 20 21 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:19:56.858703675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1304} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 29 29 29 51 51 51 F F F 2 040-059 28 28 28 50 50 50 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 26 26 26 47 47 47 T T T 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 30 30 30 52 52 52 T T T 8 160-179 21 21 21 40 40 40 F F F 9 180-199 30 30 30 52 52 52 F F F 10 200-219 30 30 30 52 52 52 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 28 28 28 50 50 50 T T T 14 280-299 25 25 25 45 45 45 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 30 30 30 52 52 52 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 28 28 28 50 50 50 T T T 22 440-459 25 25 25 45 45 45 T T T 23 460-479 24 24 24 42 42 42 T T T 24 480-499 26 26 26 47 47 47 F F F 25 500-519 30 30 30 52 52 52 T T T 26 520-539 18 18 18 35 35 35 F F F 27 540-559 27 27 27 48 48 48 T T T 28 560-579 30 30 30 52 52 52 F F F 29 580-599 28 28 28 50 50 50 T T T Max gen: 30, Max flush: 52 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 29 29 29 51 51 51 F F F 2 040-059 28 28 28 50 50 50 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 31 31 31 53 53 53 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 31 31 31 53 53 53 F F F 8 160-179 21 21 21 40 40 40 F F F 9 180-199 30 30 30 52 52 52 F F F 10 200-219 30 30 30 52 52 52 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 30 30 30 52 52 52 F F F 17 340-359 30 30 30 52 52 52 F F F 18 360-379 30 30 30 52 52 52 F F F 19 380-399 28 28 28 50 50 50 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 31 31 31 53 53 53 F F F 22 440-459 31 31 31 53 53 53 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 26 26 26 47 47 47 F F F 25 500-519 31 31 31 53 53 53 F F F 26 520-539 18 18 18 35 35 35 F F F 27 540-559 31 31 31 53 53 53 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 31, Max flush: 53 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.154081991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.199185086Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.200036223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.200083262Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.200099269Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.200113676Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312} {"msg":"Crucible b7e1c848-7446-43a8-afe7-9305b6ab4162 has session id: a46f8dc8-7221-4849-8edd-90239e852abe","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.200689445Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Upstairs opts: Upstairs UUID: b7e1c848-7446-43a8-afe7-9305b6ab4162, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.200753281Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Crucible stats registered with UUID: b7e1c848-7446-43a8-afe7-9305b6ab4162","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.200780555Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:01.20080252Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201300907Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201325271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312} {"msg":"b7e1c848-7446-43a8-afe7-9305b6ab4162 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201415371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201435516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"0","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201467959Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"0","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201501301Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"1","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201530795Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"1","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201555419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"2","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201582753Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"2","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201607977Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201634391Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201657776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"ds_connection connected from Ok(127.0.0.1:53446)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201680531Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"ds_connection connected from Ok(127.0.0.1:56820)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201705045Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"ds_connection connected from Ok(127.0.0.1:59022)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.201912988Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.202417884Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"1","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.202456105Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"0","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.202973918Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"2","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.203154767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"0","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.203191989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"1","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.203609124Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"2","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.203746663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.203772707Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[0]R flush_numbers[0..12]: [46, 51, 50, 48, 53, 45, 41, 53, 40, 52, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204159059Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[0]R generation[0..12]: [26, 29, 28, 27, 31, 25, 22, 31, 21, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204180854Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204195641Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[1]R flush_numbers[0..12]: [46, 51, 50, 48, 53, 45, 41, 53, 40, 52, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204209248Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[1]R generation[0..12]: [26, 29, 28, 27, 31, 25, 22, 31, 21, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204222915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204237022Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[2]R flush_numbers[0..12]: [46, 51, 50, 48, 53, 45, 41, 53, 40, 52, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204250499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[2]R generation[0..12]: [26, 29, 28, 27, 31, 25, 22, 31, 21, 30, 30, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204268545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204292779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204317534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204340828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:20:01.204364603Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204393286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"":"downstairs","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204418751Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204435767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"0","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204449434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"1","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.20446541Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"client":"2","":"downstairs","session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204488835Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.20451229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"b7e1c848-7446-43a8-afe7-9305b6ab4162 is now active with session: a46f8dc8-7221-4849-8edd-90239e852abe","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204535374Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204548521Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312,"session_id":"a46f8dc8-7221-4849-8edd-90239e852abe"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:01.204564978Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 178 len 24576 data: 4 5 6 6 4 3 02/30 Read block 374 len 12288 03/30 Read block 259 len 24576 04/30 Write block 494 len 4096 data: 3 05/30 Read block 31 len 8192 06/30 Write block 498 len 28672 data: 4 4 5 5 4 2 2 07/30 Write block 521 len 12288 data: 4 6 6 08/30 Read block 5 len 36864 09/30 Read block 498 len 24576 10/30 Write block 520 len 12288 data: 6 5 7 11/30 Write block 92 len 16384 data: 4 3 3 2 12/30 Write block 335 len 28672 data: 3 3 3 4 4 4 3 13/30 Write block 28 len 24576 data: 4 3 3 3 3 3 14/30 Read block 375 len 16384 15/30 Read block 513 len 36864 16/30 Read block 261 len 4096 17/30 Read block 85 len 32768 18/30 Read block 94 len 4096 19/30 Read block 173 len 24576 20/30 Flush 21/30 Write block 354 len 24576 data: 4 5 6 6 5 3 22/30 Write block 381 len 20480 data: 3 3 3 2 2 23/30 Read block 330 len 24576 24/30 Read block 445 len 12288 25/30 Read block 299 len 32768 26/30 Write block 146 len 16384 data: 7 6 6 6 27/30 Read block 511 len 16384 28/30 Read block 265 len 8192 29/30 Read block 50 len 16384 30/30 Read block 173 len 36864 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:01.223972692Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1312} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 28 28 28 50 50 50 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 32 32 32 54 54 54 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 31 31 31 53 53 53 T T T 8 160-179 32 32 32 54 54 54 F F F 9 180-199 32 32 32 54 54 54 F F F 10 200-219 30 30 30 52 52 52 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 32 32 32 54 54 54 F F F 17 340-359 32 32 32 54 54 54 T T T 18 360-379 30 30 30 52 52 52 F F F 19 380-399 28 28 28 50 50 50 T T T 20 400-419 24 24 24 43 43 43 F F F 21 420-439 31 31 31 53 53 53 F F F 22 440-459 31 31 31 53 53 53 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 32 32 32 54 54 54 F F F 26 520-539 32 32 32 54 54 54 F F F 27 540-559 31 31 31 53 53 53 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 32, Max flush: 54 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 28 28 28 50 50 50 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 32 32 32 54 54 54 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 33 33 33 55 55 55 F F F 8 160-179 32 32 32 54 54 54 F F F 9 180-199 32 32 32 54 54 54 F F F 10 200-219 30 30 30 52 52 52 F F F 11 220-239 29 29 29 51 51 51 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 30 30 30 52 52 52 F F F 16 320-339 32 32 32 54 54 54 F F F 17 340-359 33 33 33 55 55 55 F F F 18 360-379 30 30 30 52 52 52 F F F 19 380-399 33 33 33 55 55 55 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 31 31 31 53 53 53 F F F 22 440-459 31 31 31 53 53 53 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 32 32 32 54 54 54 F F F 26 520-539 32 32 32 54 54 54 F F F 27 540-559 31 31 31 53 53 53 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 33, Max flush: 55 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.4605804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.506565456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.507274655Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.507329123Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.507347699Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.507365425Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320} {"msg":"Crucible a61cfd58-57d6-4550-9f64-594b1316a29d has session id: 1ab88828-8b4c-436e-aa14-f3a28cfde353","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508060217Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Upstairs opts: Upstairs UUID: a61cfd58-57d6-4550-9f64-594b1316a29d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508103088Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Crucible stats registered with UUID: a61cfd58-57d6-4550-9f64-594b1316a29d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508121763Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:05.508139089Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508540889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508564973Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320} {"msg":"a61cfd58-57d6-4550-9f64-594b1316a29d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508637436Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508655723Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"0","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508678697Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"0","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508694664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"1","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508719088Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"1","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508742623Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"2","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508767887Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"2","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508794221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508819915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508891119Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"ds_connection connected from Ok(127.0.0.1:40612)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508917393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"ds_connection connected from Ok(127.0.0.1:61659)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508941738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"ds_connection connected from Ok(127.0.0.1:42220)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.508967712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.5095061Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"0","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.509673622Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"2","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.51012325Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"0","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.510266238Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"2","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.510487697Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"1","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.51069845Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.510767934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511028795Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"1","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[0]R flush_numbers[0..12]: [46, 54, 50, 48, 54, 45, 41, 55, 54, 54, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511522063Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[0]R generation[0..12]: [26, 32, 28, 27, 32, 25, 22, 33, 32, 32, 30, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511544238Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511559285Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[1]R flush_numbers[0..12]: [46, 54, 50, 48, 54, 45, 41, 55, 54, 54, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511572882Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[1]R generation[0..12]: [26, 32, 28, 27, 32, 25, 22, 33, 32, 32, 30, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511586559Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511600605Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[2]R flush_numbers[0..12]: [46, 54, 50, 48, 54, 45, 41, 55, 54, 54, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511620681Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[2]R generation[0..12]: [26, 32, 28, 27, 32, 25, 22, 33, 32, 32, 30, 29]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511645145Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511673099Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511693455Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.51171521Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511745733Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511770357Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"":"downstairs","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511795092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511810468Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"0","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511823615Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"1","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511843381Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"client":"2","":"downstairs","session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511867165Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.5118909Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"a61cfd58-57d6-4550-9f64-594b1316a29d is now active with session: 1ab88828-8b4c-436e-aa14-f3a28cfde353","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511908626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511920983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320,"session_id":"1ab88828-8b4c-436e-aa14-f3a28cfde353"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:05.511944547Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 186 len 36864 data: 5 5 5 6 7 8 8 7 7 02/30 Read block 393 len 4096 03/30 Read block 230 len 24576 04/30 Read block 165 len 16384 05/30 Read block 556 len 12288 06/30 Write block 172 len 16384 data: 4 5 5 5 07/30 Read block 174 len 20480 08/30 Read block 321 len 40960 09/30 Flush 10/30 Write block 379 len 32768 data: 5 4 4 4 4 3 3 3 11/30 Write block 177 len 32768 data: 4 5 6 7 7 5 4 3 12/30 Read block 490 len 16384 13/30 Read block 352 len 24576 14/30 Write block 20 len 8192 data: 5 5 15/30 Write block 319 len 24576 data: 3 4 6 5 6 4 16/30 Write block 228 len 16384 data: 4 2 2 3 17/30 Flush 18/30 Write block 321 len 32768 data: 7 6 7 5 4 3 3 3 19/30 Write block 379 len 28672 data: 6 5 5 5 5 4 4 20/30 Write block 153 len 24576 data: 3 5 6 7 10 8 21/30 Read block 191 len 12288 22/30 Flush 23/30 Write block 319 len 24576 data: 4 5 8 7 8 6 24/30 Read block 505 len 40960 25/30 Read block 410 len 8192 26/30 Write block 171 len 28672 data: 3 5 6 6 6 5 5 27/30 Read block 58 len 24576 28/30 Write block 347 len 32768 data: 3 4 5 5 5 6 6 5 29/30 Write block 1 len 4096 data: 4 30/30 Write block 34 len 24576 data: 3 2 2 3 5 5 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:05.530975807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1320} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 T T T 1 020-039 34 34 34 57 57 57 T T T 2 040-059 28 28 28 50 50 50 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 32 32 32 54 54 54 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 34 34 34 57 57 57 T T T 9 180-199 34 34 34 57 57 57 F F F 10 200-219 30 30 30 52 52 52 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 34 34 34 57 57 57 T T T 16 320-339 34 34 34 58 58 58 T T T 17 340-359 33 33 33 55 55 55 T T T 18 360-379 34 34 34 58 58 58 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 31 31 31 53 53 53 F F F 22 440-459 31 31 31 53 53 53 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 32 32 32 54 54 54 F F F 26 520-539 32 32 32 54 54 54 F F F 27 540-559 31 31 31 53 53 53 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 34, Max flush: 58 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 59 59 59 F F F 1 020-039 35 35 35 59 59 59 F F F 2 040-059 28 28 28 50 50 50 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 32 32 32 54 54 54 F F F 5 100-119 25 25 25 45 45 45 F F F 6 120-139 22 22 22 41 41 41 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 35 35 35 59 59 59 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 30 30 30 52 52 52 F F F 11 220-239 34 34 34 57 57 57 F F F 12 240-259 27 27 27 48 48 48 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 35 35 35 59 59 59 F F F 17 340-359 35 35 35 59 59 59 F F F 18 360-379 34 34 34 58 58 58 F F F 19 380-399 34 34 34 58 58 58 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 31 31 31 53 53 53 F F F 22 440-459 31 31 31 53 53 53 F F F 23 460-479 31 31 31 53 53 53 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 32 32 32 54 54 54 F F F 26 520-539 32 32 32 54 54 54 F F F 27 540-559 31 31 31 53 53 53 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 35, Max flush: 59 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.780165815Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:09.824954042Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.825369477Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.858670146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.859390582Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.859427724Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.859447499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.859462786Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328} {"msg":"Crucible b0adfb2b-ccd8-4790-8666-bdd33ef9b2f0 has session id: 44e955ef-5c3b-485e-9196-69abebc7c0b0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860048753Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Upstairs opts: Upstairs UUID: b0adfb2b-ccd8-4790-8666-bdd33ef9b2f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860082276Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Crucible stats registered with UUID: b0adfb2b-ccd8-4790-8666-bdd33ef9b2f0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860097132Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:09.860109659Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860610626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.86063531Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328} {"msg":"b0adfb2b-ccd8-4790-8666-bdd33ef9b2f0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860736857Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860754493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"0","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860777838Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"0","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860791545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"1","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860805762Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"1","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860819139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"2","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860843543Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"2","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860928334Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.860967185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.86103322Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39537)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.861152093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39923)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.861185886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:41057)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.8612103Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.861759506Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"1","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.861794048Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"2","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.861894535Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"0","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.862514155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"1","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.862552646Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"2","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.86258064Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"0","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.86311015Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863136004Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[0]R flush_numbers[0..12]: [59, 59, 50, 48, 54, 45, 41, 58, 59, 57, 52, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863160698Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[0]R generation[0..12]: [35, 35, 28, 27, 32, 25, 22, 34, 35, 34, 30, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863175885Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863190052Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[1]R flush_numbers[0..12]: [59, 59, 50, 48, 54, 45, 41, 58, 59, 57, 52, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863203409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[1]R generation[0..12]: [35, 35, 28, 27, 32, 25, 22, 34, 35, 34, 30, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863216816Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863230363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[2]R flush_numbers[0..12]: [59, 59, 50, 48, 54, 45, 41, 58, 59, 57, 52, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.86324358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[2]R generation[0..12]: [35, 35, 28, 27, 32, 25, 22, 34, 35, 34, 30, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863256847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863270444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863283021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863295998Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863308665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863321312Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"":"downstairs","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863334019Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863345816Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"0","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863397025Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"1","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863411512Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"client":"2","":"downstairs","session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863424519Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863436446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"b0adfb2b-ccd8-4790-8666-bdd33ef9b2f0 is now active with session: 44e955ef-5c3b-485e-9196-69abebc7c0b0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863448913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863460661Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328,"session_id":"44e955ef-5c3b-485e-9196-69abebc7c0b0"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:09.863472578Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 40960 data: 4 6 7 6 7 5 5 4 5 4 02/30 Read block 302 len 12288 03/30 Write block 518 len 20480 data: 6 8 7 6 8 04/30 Flush 05/30 Write block 448 len 12288 data: 4 4 6 06/30 Write block 373 len 28672 data: 3 3 3 4 4 5 7 07/30 Read block 261 len 28672 08/30 Write block 550 len 8192 data: 4 5 09/30 Write block 241 len 36864 data: 5 6 5 4 3 2 2 3 3 10/30 Flush 11/30 Write block 235 len 16384 data: 7 8 7 8 12/30 Write block 130 len 36864 data: 3 2 2 2 2 2 3 3 3 13/30 Read block 426 len 24576 14/30 Read block 428 len 12288 15/30 Write block 159 len 36864 data: 9 8 7 7 7 5 4 5 4 16/30 Write block 43 len 12288 data: 4 4 4 17/30 Write block 393 len 16384 data: 4 4 4 5 18/30 Read block 277 len 28672 19/30 Read block 375 len 12288 20/30 Read block 205 len 4096 21/30 Read block 284 len 8192 22/30 Write block 115 len 16384 data: 4 5 4 3 23/30 Read block 237 len 36864 24/30 Write block 478 len 8192 data: 2 2 25/30 Read block 143 len 28672 26/30 Read block 491 len 16384 27/30 Read block 11 len 12288 28/30 Read block 164 len 4096 29/30 Read block 29 len 24576 30/30 Read block 593 len 20480 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 4 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 20 19 20 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:09.883382869Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1328} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 59 59 59 F F F 1 020-039 35 35 35 59 59 59 F F F 2 040-059 28 28 28 50 50 50 T T T 3 060-079 27 27 27 48 48 48 F F F 4 080-099 32 32 32 54 54 54 F F F 5 100-119 25 25 25 45 45 45 T T T 6 120-139 22 22 22 41 41 41 T T T 7 140-159 34 34 34 58 58 58 T T T 8 160-179 35 35 35 59 59 59 T T T 9 180-199 34 34 34 57 57 57 F F F 10 200-219 30 30 30 52 52 52 F F F 11 220-239 36 36 36 60 60 60 T T T 12 240-259 36 36 36 61 61 61 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 35 35 35 59 59 59 F F F 17 340-359 35 35 35 59 59 59 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 34 34 34 58 58 58 T T T 20 400-419 24 24 24 43 43 43 F F F 21 420-439 31 31 31 53 53 53 F F F 22 440-459 36 36 36 61 61 61 F F F 23 460-479 31 31 31 53 53 53 T T T 24 480-499 32 32 32 54 54 54 F F F 25 500-519 36 36 36 60 60 60 F F F 26 520-539 36 36 36 60 60 60 F F F 27 540-559 36 36 36 61 61 61 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 36, Max flush: 61 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 35 35 35 59 59 59 F F F 1 020-039 35 35 35 59 59 59 F F F 2 040-059 37 37 37 62 62 62 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 32 32 32 54 54 54 F F F 5 100-119 37 37 37 62 62 62 F F F 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 37 37 37 62 62 62 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 30 30 30 52 52 52 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 36 36 36 61 61 61 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 35 35 35 59 59 59 F F F 17 340-359 35 35 35 59 59 59 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 31 31 31 53 53 53 F F F 22 440-459 36 36 36 61 61 61 F F F 23 460-479 37 37 37 62 62 62 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 36 36 36 60 60 60 F F F 26 520-539 36 36 36 60 60 60 F F F 27 540-559 36 36 36 61 61 61 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 37, Max flush: 62 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.173889939Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.218545136Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.219199357Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.219246837Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.219261513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.21927623Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336} {"msg":"Crucible 4699af07-bc1f-49b6-b2ae-16f53856b0d8 has session id: ea23f1c2-588b-4bd7-ba95-b4532d44a20f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.219852729Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Upstairs opts: Upstairs UUID: 4699af07-bc1f-49b6-b2ae-16f53856b0d8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.21989267Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Crucible stats registered with UUID: 4699af07-bc1f-49b6-b2ae-16f53856b0d8","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.219920564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:14.219942359Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220400765Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.2204234Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336} {"msg":"4699af07-bc1f-49b6-b2ae-16f53856b0d8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.22051342Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220532166Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"0","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.22055544Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"0","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220569717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"1","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220583674Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"1","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220597081Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"2","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220611168Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"2","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220623885Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220637162Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220686021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50128)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220884576Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47367)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.220922087Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61344)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.22095118Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.221620639Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"2","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.221656271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"0","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.221914142Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"1","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.222381027Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"2","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.222481924Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"0","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.222513167Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"1","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.222954497Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.222988639Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[0]R flush_numbers[0..12]: [59, 59, 62, 48, 54, 62, 62, 62, 62, 57, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223009125Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[0]R generation[0..12]: [35, 35, 37, 27, 32, 37, 37, 37, 37, 34, 30, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223024691Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223039268Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[1]R flush_numbers[0..12]: [59, 59, 62, 48, 54, 62, 62, 62, 62, 57, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223053224Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[1]R generation[0..12]: [35, 35, 37, 27, 32, 37, 37, 37, 37, 34, 30, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223066552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223080268Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[2]R flush_numbers[0..12]: [59, 59, 62, 48, 54, 62, 62, 62, 62, 57, 52, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223093375Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[2]R generation[0..12]: [35, 35, 37, 27, 32, 37, 37, 37, 37, 34, 30, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223106492Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223124099Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223136985Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223149803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.22316234Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223175127Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"":"downstairs","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223188064Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223199791Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"0","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:20:14.223213128Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"1","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223232614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"client":"2","":"downstairs","session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223246871Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223259068Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"4699af07-bc1f-49b6-b2ae-16f53856b0d8 is now active with session: ea23f1c2-588b-4bd7-ba95-b4532d44a20f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223271685Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.223283542Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336,"session_id":"ea23f1c2-588b-4bd7-ba95-b4532d44a20f"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:14.22329548Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 420 len 4096 data: 4 02/30 Write block 26 len 32768 data: 4 4 5 4 4 4 4 4 03/30 Write block 516 len 8192 data: 7 6 04/30 Write block 35 len 4096 data: 3 05/30 Write block 549 len 4096 data: 3 06/30 Read block 519 len 40960 07/30 Write block 523 len 36864 data: 7 6 6 5 3 3 3 3 3 08/30 Write block 447 len 4096 data: 3 09/30 Read block 571 len 32768 10/30 Read block 54 len 8192 11/30 Flush 12/30 Flush 13/30 Write block 241 len 4096 data: 6 14/30 Read block 16 len 28672 15/30 Write block 0 len 8192 data: 2 5 16/30 Write block 329 len 32768 data: 3 3 2 2 3 3 4 4 17/30 Write block 201 len 8192 data: 5 5 18/30 Read block 78 len 4096 19/30 Flush 20/30 Write block 443 len 12288 data: 3 3 2 21/30 Read block 555 len 4096 22/30 Write block 349 len 12288 data: 6 6 6 23/30 Flush 24/30 Flush 25/30 Write block 529 len 4096 data: 4 26/30 Read block 584 len 28672 27/30 Read block 382 len 36864 28/30 Write block 528 len 36864 data: 4 5 4 4 2 2 2 2 4 29/30 Write block 240 len 20480 data: 6 7 7 6 5 30/30 Write block 459 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:14.239084164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1336} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 65 65 65 F F F 1 020-039 38 38 38 63 63 63 F F F 2 040-059 37 37 37 62 62 62 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 32 32 32 54 54 54 F F F 5 100-119 37 37 37 62 62 62 F F F 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 37 37 37 62 62 62 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 38 38 38 65 65 65 T T T 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 38 38 38 65 65 65 F F F 17 340-359 38 38 38 66 66 66 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 38 38 38 66 66 66 T T T 23 460-479 37 37 37 62 62 62 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 38 38 38 63 63 63 F F F 26 520-539 38 38 38 63 63 63 T T T 27 540-559 38 38 38 63 63 63 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 38, Max flush: 66 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 65 65 65 F F F 1 020-039 38 38 38 63 63 63 F F F 2 040-059 37 37 37 62 62 62 F F F 3 060-079 27 27 27 48 48 48 F F F 4 080-099 32 32 32 54 54 54 F F F 5 100-119 37 37 37 62 62 62 F F F 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 37 37 37 62 62 62 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 38 38 38 65 65 65 F F F 11 220-239 37 37 37 62 62 62 F F F 12 240-259 39 39 39 67 67 67 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 38 38 38 65 65 65 F F F 17 340-359 38 38 38 66 66 66 F F F 18 360-379 36 36 36 61 61 61 F F F 19 380-399 37 37 37 62 62 62 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 38 38 38 63 63 63 F F F 22 440-459 39 39 39 67 67 67 F F F 23 460-479 37 37 37 62 62 62 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 38 38 38 63 63 63 F F F 26 520-539 39 39 39 67 67 67 F F F 27 540-559 38 38 38 63 63 63 F F F 28 560-579 30 30 30 52 52 52 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 39, Max flush: 67 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.471218506Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.517328253Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.518133331Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.518183069Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.518201655Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.518219151Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344} {"msg":"Crucible 0f6d32ac-de5c-49bd-a1da-64541fb321a1 has session id: c53c2f11-818a-4094-8112-912b3f11d404","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.518878672Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Upstairs opts: Upstairs UUID: 0f6d32ac-de5c-49bd-a1da-64541fb321a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.518936609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Crucible stats registered with UUID: 0f6d32ac-de5c-49bd-a1da-64541fb321a1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.518972571Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:18.519022419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519519757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519556758Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344} {"msg":"0f6d32ac-de5c-49bd-a1da-64541fb321a1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519646968Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519687259Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"0","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519723761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"0","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519749595Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"1","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519772399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"1","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519791095Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"2","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.51981258Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"2","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519837395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519862089Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519887683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"ds_connection connected from Ok(127.0.0.1:37510)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.519958867Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"ds_connection connected from Ok(127.0.0.1:64231)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.520039989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"ds_connection connected from Ok(127.0.0.1:56065)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.52008012Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.520793238Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"0","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.520820502Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"1","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.520835819Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"2","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.521383015Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"0","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.521442441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"1","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.521511276Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"2","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.521956365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522086875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[0]R flush_numbers[0..12]: [65, 63, 62, 48, 54, 62, 62, 62, 62, 57, 65, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522162068Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[0]R generation[0..12]: [38, 38, 37, 27, 32, 37, 37, 37, 37, 34, 38, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522183673Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.52219918Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[1]R flush_numbers[0..12]: [65, 63, 62, 48, 54, 62, 62, 62, 62, 57, 65, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522213387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[1]R generation[0..12]: [38, 38, 37, 27, 32, 37, 37, 37, 37, 34, 38, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522227023Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522245449Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[2]R flush_numbers[0..12]: [65, 63, 62, 48, 54, 62, 62, 62, 62, 57, 65, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522259706Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[2]R generation[0..12]: [38, 38, 37, 27, 32, 37, 37, 37, 37, 34, 38, 37]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522273043Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.5222868Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522299537Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522312454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522325151Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522337878Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"":"downstairs","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522350695Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522368582Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"0","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522381699Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"1","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522394525Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"client":"2","":"downstairs","session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522407463Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.52241951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"0f6d32ac-de5c-49bd-a1da-64541fb321a1 is now active with session: c53c2f11-818a-4094-8112-912b3f11d404","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522432037Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522446264Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344,"session_id":"c53c2f11-818a-4094-8112-912b3f11d404"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:18.522458361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 101 len 12288 data: 3 3 3 02/30 Write block 229 len 16384 data: 3 3 4 3 03/30 Read block 441 len 32768 04/30 Write block 432 len 28672 data: 4 5 6 3 2 2 2 05/30 Write block 94 len 8192 data: 4 3 06/30 Write block 393 len 24576 data: 5 5 5 6 5 5 07/30 Flush 08/30 Write block 432 len 40960 data: 5 6 7 4 3 3 3 2 2 3 09/30 Write block 168 len 28672 data: 2 2 2 4 6 7 7 10/30 Read block 301 len 4096 11/30 Write block 200 len 24576 data: 6 6 6 3 2 3 12/30 Read block 324 len 8192 13/30 Write block 441 len 12288 data: 4 3 4 14/30 Write block 209 len 12288 data: 4 2 3 15/30 Write block 339 len 24576 data: 5 5 4 2 2 2 16/30 Write block 70 len 20480 data: 3 3 3 3 3 17/30 Read block 434 len 40960 18/30 Write block 431 len 40960 data: 4 6 7 8 5 4 4 4 3 3 19/30 Read block 163 len 32768 20/30 Read block 441 len 32768 21/30 Read block 175 len 20480 22/30 Flush 23/30 Write block 566 len 28672 data: 2 3 3 4 3 3 3 24/30 Read block 458 len 24576 25/30 Write block 372 len 16384 data: 3 4 4 4 26/30 Read block 562 len 40960 27/30 Write block 326 len 20480 data: 4 4 4 4 4 28/30 Write block 169 len 4096 data: 3 29/30 Write block 465 len 16384 data: 3 3 3 4 30/30 Read block 88 len 12288 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:18.54114039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1344} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 65 65 65 F F F 1 020-039 38 38 38 63 63 63 F F F 2 040-059 37 37 37 62 62 62 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 40 40 40 68 68 68 F F F 5 100-119 40 40 40 68 68 68 F F F 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 40 40 40 69 69 69 T T T 9 180-199 34 34 34 57 57 57 F F F 10 200-219 40 40 40 69 69 69 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 39 39 39 67 67 67 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 40 40 40 69 69 69 T T T 17 340-359 40 40 40 69 69 69 F F F 18 360-379 36 36 36 61 61 61 T T T 19 380-399 40 40 40 68 68 68 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 40 40 40 69 69 69 F F F 22 440-459 40 40 40 69 69 69 F F F 23 460-479 37 37 37 62 62 62 T T T 24 480-499 32 32 32 54 54 54 F F F 25 500-519 38 38 38 63 63 63 F F F 26 520-539 39 39 39 67 67 67 F F F 27 540-559 38 38 38 63 63 63 F F F 28 560-579 30 30 30 52 52 52 T T T 29 580-599 31 31 31 53 53 53 F F F Max gen: 40, Max flush: 69 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 65 65 65 F F F 1 020-039 38 38 38 63 63 63 F F F 2 040-059 37 37 37 62 62 62 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 40 40 40 68 68 68 F F F 5 100-119 40 40 40 68 68 68 F F F 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 41 41 41 70 70 70 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 40 40 40 69 69 69 F F F 11 220-239 40 40 40 68 68 68 F F F 12 240-259 39 39 39 67 67 67 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 41 41 41 70 70 70 F F F 17 340-359 40 40 40 69 69 69 F F F 18 360-379 41 41 41 70 70 70 F F F 19 380-399 40 40 40 68 68 68 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 40 40 40 69 69 69 F F F 22 440-459 40 40 40 69 69 69 F F F 23 460-479 41 41 41 70 70 70 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 38 38 38 63 63 63 F F F 26 520-539 39 39 39 67 67 67 F F F 27 540-559 38 38 38 63 63 63 F F F 28 560-579 41 41 41 70 70 70 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 41, Max flush: 70 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.779289159Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.824860358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.825664446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.825708486Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.825728841Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.825743658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352} {"msg":"Crucible eacc5b36-1329-4a47-b863-3ad5fc333c50 has session id: 54f8a0b0-1775-4333-bdc0-2f0d213fa8fe","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.8263527Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Upstairs opts: Upstairs UUID: eacc5b36-1329-4a47-b863-3ad5fc333c50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.826398249Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Crucible stats registered with UUID: eacc5b36-1329-4a47-b863-3ad5fc333c50","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.826422174Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:22.82644196Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.826911873Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.826938457Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352} {"msg":"eacc5b36-1329-4a47-b863-3ad5fc333c50 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827025068Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827042783Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"0","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827066868Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"0","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827080835Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"1","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827094812Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"1","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827107559Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"2","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827124575Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"2","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827152429Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827178013Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827203717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:52142)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827449662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:55947)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827486173Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:40568)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.827511737Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.828019602Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"0","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.828052105Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"1","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.828573827Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"2","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.8286934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"0","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.828723063Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"1","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829161684Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"2","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829348801Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829375615Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[0]R flush_numbers[0..12]: [65, 63, 62, 69, 68, 68, 62, 62, 70, 57, 69, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829717288Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[0]R generation[0..12]: [38, 38, 37, 40, 40, 40, 37, 37, 41, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829754069Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829779084Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[1]R flush_numbers[0..12]: [65, 63, 62, 69, 68, 68, 62, 62, 70, 57, 69, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829801589Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[1]R generation[0..12]: [38, 38, 37, 40, 40, 40, 37, 37, 41, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829824784Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829850528Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[2]R flush_numbers[0..12]: [65, 63, 62, 69, 68, 68, 62, 62, 70, 57, 69, 68]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829876202Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[2]R generation[0..12]: [38, 38, 37, 40, 40, 40, 37, 37, 41, 34, 40, 40]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829900836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.8299279Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:20:22.829954294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.829981148Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830007232Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830032217Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"":"downstairs","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830058341Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830082375Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"0","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830113808Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"1","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830141522Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"client":"2","":"downstairs","session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"All required reconciliation work is completed","v":0,Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "name":"crucible","level":30,"time":"2025-01-15T05:20:22.830172345Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830197139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"eacc5b36-1329-4a47-b863-3ad5fc333c50 is now active with session: 54f8a0b0-1775-4333-bdc0-2f0d213fa8fe","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830220004Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830232931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352,"session_id":"54f8a0b0-1775-4333-bdc0-2f0d213fa8fe"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:22.830245218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 43 len 24576 data: 5 5 5 3 3 3 02/30 Write block 466 len 24576 data: 4 4 5 4 4 5 03/30 Read block 438 len 20480 04/30 Write block 521 len 4096 data: 7 05/30 Write block 163 len 12288 data: 8 6 5 06/30 Read block 260 len 40960 07/30 Read block 456 len 36864 08/30 Read block 487 len 20480 09/30 Flush 10/30 Write block 445 len 40960 data: 3 2 4 5 5 7 6 6 6 5 11/30 Read block 500 len 20480 12/30 Write block 100 len 40960 data: 2 4 4 4 3 3 4 4 5 3 13/30 Write block 89 len 36864 data: 4 4 4 5 4 5 4 2 3 14/30 Read block 297 len 28672 15/30 Read block 33 len 24576 16/30 Read block 452 len 12288 17/30 Write block 575 len 8192 data: 3 4 18/30 Write block 464 len 40960 data: 5 4 5 5 6 5 5 6 5 3 19/30 Read block 201 len 32768 20/30 Write block 225 len 28672 data: 5 5 5 5 4 4 5 21/30 Read block 350 len 40960 22/30 Read block 179 len 32768 23/30 Write block 423 len 24576 data: 4 4 5 5 4 4 24/30 Write block 335 len 28672 data: 5 5 4 5 6 6 5 25/30 Read block 51 len 40960 26/30 Write block 379 len 28672 data: 8 6 6 6 6 5 5 27/30 Read block 414 len 16384 28/30 Read block 278 len 8192 29/30 Read block 13 len 32768 30/30 Write block 202 len 12288 data: 7 4 3 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 10 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:22.851627316Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1352} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 65 65 65 F F F 1 020-039 38 38 38 63 63 63 F F F 2 040-059 42 42 42 71 71 71 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 40 40 40 68 68 68 T T T 5 100-119 40 40 40 68 68 68 T T T 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 42 42 42 71 71 71 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 40 40 40 69 69 69 T T T 11 220-239 40 40 40 68 68 68 T T T 12 240-259 39 39 39 67 67 67 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 41 41 41 70 70 70 T T T 17 340-359 40 40 40 69 69 69 T T T 18 360-379 41 41 41 70 70 70 T T T 19 380-399 40 40 40 68 68 68 T T T 20 400-419 24 24 24 43 43 43 F F F 21 420-439 40 40 40 69 69 69 T T T 22 440-459 40 40 40 69 69 69 T T T 23 460-479 42 42 42 71 71 71 T T T 24 480-499 32 32 32 54 54 54 F F F 25 500-519 38 38 38 63 63 63 F F F 26 520-539 42 42 42 71 71 71 F F F 27 540-559 38 38 38 63 63 63 F F F 28 560-579 41 41 41 70 70 70 T T T 29 580-599 31 31 31 53 53 53 F F F Max gen: 42, Max flush: 71 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 65 65 65 F F F 1 020-039 38 38 38 63 63 63 F F F 2 040-059 42 42 42 71 71 71 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 43 43 43 72 72 72 F F F 5 100-119 43 43 43 72 72 72 F F F 6 120-139 37 37 37 62 62 62 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 42 42 42 71 71 71 F F F 9 180-199 34 34 34 57 57 57 F F F 10 200-219 43 43 43 72 72 72 F F F 11 220-239 43 43 43 72 72 72 F F F 12 240-259 39 39 39 67 67 67 F F F 13 260-279 31 31 31 53 53 53 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 43 43 43 72 72 72 F F F 17 340-359 43 43 43 72 72 72 F F F 18 360-379 43 43 43 72 72 72 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 43 43 43 72 72 72 F F F 22 440-459 43 43 43 72 72 72 F F F 23 460-479 43 43 43 72 72 72 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 38 38 38 63 63 63 F F F 26 520-539 42 42 42 71 71 71 F F F 27 540-559 38 38 38 63 63 63 F F F 28 560-579 43 43 43 72 72 72 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 43, Max flush: 72 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.130966936Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:27.174780214Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.175172465Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.208348731Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.208965961Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.209003203Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.209018929Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.209035805Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360} {"msg":"Crucible a7c6bb9d-3105-4e03-9687-eb4ef4a0168c has session id: 3ec5126c-103b-43f8-8ba9-612c22d403ea","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.209647787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Upstairs opts: Upstairs UUID: a7c6bb9d-3105-4e03-9687-eb4ef4a0168c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.209733817Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Crucible stats registered with UUID: a7c6bb9d-3105-4e03-9687-eb4ef4a0168c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.20976413Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:27.209790484Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210173627Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210197712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360} {"msg":"a7c6bb9d-3105-4e03-9687-eb4ef4a0168c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210260348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210278714Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"0","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210301718Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"0","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210315635Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"1","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210331082Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"1","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210344169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"2","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210358146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"2","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210370853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.21038439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210408894Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:49611)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210443836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:52495)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.21047184Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:54261)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.210500593Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.211263201Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"2","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.211292934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"1","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.211414177Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"0","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.211872873Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"1","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.211913664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"2","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.211964922Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"0","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212473637Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.21250206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[0]R flush_numbers[0..12]: [65, 63, 71, 69, 72, 72, 62, 62, 71, 57, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212544001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[0]R generation[0..12]: [38, 38, 42, 40, 43, 43, 37, 37, 42, 34, 43, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212560657Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212574914Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[1]R flush_numbers[0..12]: [65, 63, 71, 69, 72, 72, 62, 62, 71, 57, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.21258854Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[1]R generation[0..12]: [38, 38, 42, 40, 43, 43, 37, 37, 42, 34, 43, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212601828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212615314Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[2]R flush_numbers[0..12]: [65, 63, 71, 69, 72, 72, 62, 62, 71, 57, 72, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.21263265Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[2]R generation[0..12]: [38, 38, 42, 40, 43, 43, 37, 37, 42, 34, 43, 43]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212646218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212659764Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212672342Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212685289Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212697766Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212710283Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"":"downstairs","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.21272313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:20:27.212734738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"0","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.21281222Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"1","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212828266Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"client":"2","":"downstairs","session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212841423Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.21285348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"a7c6bb9d-3105-4e03-9687-eb4ef4a0168c is now active with session: 3ec5126c-103b-43f8-8ba9-612c22d403ea","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212865958Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212877595Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360,"session_id":"3ec5126c-103b-43f8-8ba9-612c22d403ea"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:27.212893491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 185 len 24576 data: 4 6 6 6 7 8 02/30 Write block 107 len 20480 data: 5 6 4 3 4 03/30 Write block 266 len 24576 data: 7 6 7 6 5 4 04/30 Write block 209 len 20480 data: 5 3 4 4 4 05/30 Read block 30 len 16384 06/30 Read block 538 len 36864 07/30 Read block 272 len 16384 08/30 Write block 447 len 24576 data: 5 6 6 8 7 7 09/30 Read block 495 len 36864 10/30 Read block 413 len 4096 11/30 Read block 127 len 8192 12/30 Read block 400 len 28672 13/30 Read block 590 len 20480 14/30 Write block 17 len 40960 data: 4 4 4 6 6 4 3 3 3 5 15/30 Write block 503 len 8192 data: 3 3 16/30 Flush 17/30 Flush 18/30 Write block 217 len 28672 data: 5 6 5 5 5 5 7 19/30 Read block 440 len 12288 20/30 Flush 21/30 Write block 373 len 20480 data: 5 5 5 5 5 22/30 Write block 120 len 20480 data: 2 3 3 3 4 23/30 Write block 352 len 20480 data: 7 7 6 6 7 24/30 Read block 506 len 40960 25/30 Read block 62 len 20480 26/30 Read block 136 len 20480 27/30 Read block 470 len 24576 28/30 Read block 147 len 24576 29/30 Write block 192 len 4096 data: 9 30/30 Write block 524 len 28672 data: 7 7 6 4 5 6 5 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:27.232021649Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1360} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 42 42 42 71 71 71 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 43 43 43 72 72 72 F F F 5 100-119 44 44 44 73 73 73 F F F 6 120-139 37 37 37 62 62 62 T T T 7 140-159 37 37 37 62 62 62 F F F 8 160-179 42 42 42 71 71 71 F F F 9 180-199 44 44 44 73 73 73 T T T 10 200-219 44 44 44 75 75 75 F F F 11 220-239 44 44 44 75 75 75 F F F 12 240-259 39 39 39 67 67 67 F F F 13 260-279 44 44 44 73 73 73 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 43 43 43 72 72 72 F F F 17 340-359 43 43 43 72 72 72 T T T 18 360-379 43 43 43 72 72 72 T T T 19 380-399 43 43 43 72 72 72 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 43 43 43 72 72 72 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 43 43 43 72 72 72 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 44 44 44 73 73 73 F F F 26 520-539 42 42 42 71 71 71 T T T 27 540-559 38 38 38 63 63 63 F F F 28 560-579 43 43 43 72 72 72 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 44, Max flush: 75 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 42 42 42 71 71 71 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 43 43 43 72 72 72 F F F 5 100-119 44 44 44 73 73 73 F F F 6 120-139 45 45 45 76 76 76 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 42 42 42 71 71 71 F F F 9 180-199 45 45 45 76 76 76 F F F 10 200-219 44 44 44 75 75 75 F F F 11 220-239 44 44 44 75 75 75 F F F 12 240-259 39 39 39 67 67 67 F F F 13 260-279 44 44 44 73 73 73 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 43 43 43 72 72 72 F F F 17 340-359 45 45 45 76 76 76 F F F 18 360-379 45 45 45 76 76 76 F F F 19 380-399 43 43 43 72 72 72 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 43 43 43 72 72 72 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 43 43 43 72 72 72 F F F 24 480-499 32 32 32 54 54 54 F F F 25 500-519 44 44 44 73 73 73 F F F 26 520-539 45 45 45 76 76 76 F F F 27 540-559 38 38 38 63 63 63 F F F 28 560-579 43 43 43 72 72 72 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 45, Max flush: 76 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.508036662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.553594914Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.554239409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.554282919Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.554297785Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.554312012Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368} {"msg":"Crucible dc35b5a6-9aa5-4a55-8ff4-621496c7e1e2 has session id: 8a26fb7f-f748-4b5a-a527-0a540168c3eb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.554943789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Upstairs opts: Upstairs UUID: dc35b5a6-9aa5-4a55-8ff4-621496c7e1e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.554999986Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Crucible stats registered with UUID: dc35b5a6-9aa5-4a55-8ff4-621496c7e1e2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.55502676Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:31.555041327Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555728491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555754325Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368} {"msg":"dc35b5a6-9aa5-4a55-8ff4-621496c7e1e2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555806064Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555833547Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"0","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555860341Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"0","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555876208Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"1","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555890375Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"1","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555903472Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"2","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555917488Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"2","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555930795Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555948451Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.555963218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:64553)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.556146077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:50421)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.556268189Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:49980)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.556305141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.556796349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"0","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.556953814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"1","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.556979358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"2","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.557481494Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"0","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.557514007Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"2","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.557628181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"1","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558043397Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558154822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[0]R flush_numbers[0..12]: [73, 73, 71, 69, 72, 73, 76, 62, 71, 76, 75, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558219607Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[0]R generation[0..12]: [44, 44, 42, 40, 43, 44, 45, 37, 42, 45, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558239403Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558257259Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[1]R flush_numbers[0..12]: [73, 73, 71, 69, 72, 73, 76, 62, 71, 76, 75, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558271055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[1]R generation[0..12]: [44, 44, 42, 40, 43, 44, 45, 37, 42, 45, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558284742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558298739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[2]R flush_numbers[0..12]: [73, 73, 71, 69, 72, 73, 76, 62, 71, 76, 75, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558312076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[2]R generation[0..12]: [44, 44, 42, 40, 43, 44, 45, 37, 42, 45, 44, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558325253Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.55833869Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558351197Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558364064Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558376392Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558389049Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"":"downstairs","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:20:31.558402296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558416662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"0","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558430189Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"1","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558445436Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"client":"2","":"downstairs","session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558458983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.55847116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"dc35b5a6-9aa5-4a55-8ff4-621496c7e1e2 is now active with session: 8a26fb7f-f748-4b5a-a527-0a540168c3eb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558484647Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558496314Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368,"session_id":"8a26fb7f-f748-4b5a-a527-0a540168c3eb"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:31.558508331Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 563 len 40960 data: 2 2 2 3 4 4 5 4 4 4 02/30 Read block 340 len 8192 03/30 Read block 222 len 36864 04/30 Read block 66 len 12288 05/30 Read block 140 len 28672 06/30 Read block 568 len 28672 07/30 Read block 154 len 28672 08/30 Read block 490 len 40960 09/30 Read block 89 len 24576 10/30 Read block 357 len 28672 11/30 Read block 156 len 24576 12/30 Flush 13/30 Write block 89 len 20480 data: 5 5 5 6 5 14/30 Write block 375 len 40960 data: 6 6 6 6 9 7 7 7 7 6 15/30 Write block 160 len 12288 data: 9 8 8 16/30 Read block 578 len 32768 17/30 Write block 249 len 16384 data: 4 3 5 3 18/30 Read block 373 len 24576 19/30 Read block 49 len 36864 20/30 Read block 236 len 12288 21/30 Read block 266 len 12288 22/30 Write block 195 len 24576 data: 6 6 5 5 6 7 23/30 Read block 428 len 20480 24/30 Write block 479 len 40960 data: 3 3 4 4 4 4 4 4 6 7 25/30 Read block 235 len 20480 26/30 Write block 202 len 28672 data: 8 5 4 4 3 4 4 27/30 Write block 5 len 12288 data: 4 4 4 28/30 Write block 45 len 8192 data: 6 4 29/30 Read block 190 len 12288 30/30 Read block 409 len 28672 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 5 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 18 18 17 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:31.579030164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1368} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 73 73 73 T T T 1 020-039 44 44 44 73 73 73 F F F 2 040-059 42 42 42 71 71 71 T T T 3 060-079 40 40 40 69 69 69 F F F 4 080-099 43 43 43 72 72 72 T T T 5 100-119 44 44 44 73 73 73 F F F 6 120-139 45 45 45 76 76 76 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 42 42 42 71 71 71 T T T 9 180-199 45 45 45 76 76 76 T T T 10 200-219 44 44 44 75 75 75 T T T 11 220-239 44 44 44 75 75 75 F F F 12 240-259 39 39 39 67 67 67 T T T 13 260-279 44 44 44 73 73 73 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 43 43 43 72 72 72 F F F 17 340-359 45 45 45 76 76 76 F F F 18 360-379 45 45 45 76 76 76 T T T 19 380-399 43 43 43 72 72 72 T T T 20 400-419 24 24 24 43 43 43 F F F 21 420-439 43 43 43 72 72 72 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 43 43 43 72 72 72 T T T 24 480-499 32 32 32 54 54 54 T T T 25 500-519 44 44 44 73 73 73 F F F 26 520-539 45 45 45 76 76 76 F F F 27 540-559 38 38 38 63 63 63 F F F 28 560-579 46 46 46 77 77 77 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 46, Max flush: 77 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 78 78 78 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 47 47 47 78 78 78 F F F 3 060-079 40 40 40 69 69 69 F F F 4 080-099 47 47 47 78 78 78 F F F 5 100-119 44 44 44 73 73 73 F F F 6 120-139 45 45 45 76 76 76 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 47 47 47 78 78 78 F F F 10 200-219 47 47 47 78 78 78 F F F 11 220-239 44 44 44 75 75 75 F F F 12 240-259 47 47 47 78 78 78 F F F 13 260-279 44 44 44 73 73 73 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 F F F 16 320-339 43 43 43 72 72 72 F F F 17 340-359 45 45 45 76 76 76 F F F 18 360-379 47 47 47 78 78 78 F F F 19 380-399 47 47 47 78 78 78 F F F 20 400-419 24 24 24 43 43 43 F F F 21 420-439 43 43 43 72 72 72 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 47 47 47 78 78 78 F F F 24 480-499 47 47 47 78 78 78 F F F 25 500-519 44 44 44 73 73 73 F F F 26 520-539 45 45 45 76 76 76 F F F 27 540-559 38 38 38 63 63 63 F F F 28 560-579 46 46 46 77 77 77 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 47, Max flush: 78 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.856760599Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:35.900716484Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.901105975Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.933453909Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.934054623Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.934098983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.934116109Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.934133655Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376} {"msg":"Crucible ed38814d-0819-4e78-af69-af30cc2bbeb1 has session id: a0542573-833a-4dbb-8389-b325b521cd4a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.934792276Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Upstairs opts: Upstairs UUID: ed38814d-0819-4e78-af69-af30cc2bbeb1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.934835147Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Crucible stats registered with UUID: ed38814d-0819-4e78-af69-af30cc2bbeb1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.934857601Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:35.934877597Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935359997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935387641Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376} {"msg":"ed38814d-0819-4e78-af69-af30cc2bbeb1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935450487Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935469303Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"0","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935491968Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"0","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935505924Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"1","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935520221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"1","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935533208Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"2","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935549965Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"2","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935572969Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935597074Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935619899Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:41996)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.93565996Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46504)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935687003Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64802)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.935712718Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.936448071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"1","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.936475275Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"2","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.936494891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"0","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937092815Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"1","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937228884Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"0","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937263586Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"2","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937624065Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937828269Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[0]R flush_numbers[0..12]: [78, 73, 78, 69, 78, 73, 76, 62, 78, 78, 78, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937879787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[0]R generation[0..12]: [47, 44, 47, 40, 47, 44, 45, 37, 47, 47, 47, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937899712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937915189Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[1]R flush_numbers[0..12]: [78, 73, 78, 69, 78, 73, 76, 62, 78, 78, 78, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937928986Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[1]R generation[0..12]: [47, 44, 47, 40, 47, 44, 45, 37, 47, 47, 47, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937942373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.93795618Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[2]R flush_numbers[0..12]: [78, 73, 78, 69, 78, 73, 76, 62, 78, 78, 78, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937969646Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[2]R generation[0..12]: [47, 44, 47, 40, 47, 44, 45, 37, 47, 47, 47, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.937982914Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938001319Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938014806Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938028003Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.93804456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938067684Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"":"downstairs","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:20:35.938090549Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938108755Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"0","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938121972Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"1","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938134759Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"client":"2","":"downstairs","session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938147596Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938159614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"ed38814d-0819-4e78-af69-af30cc2bbeb1 is now active with session: a0542573-833a-4dbb-8389-b325b521cd4a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938172181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938183878Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376,"session_id":"a0542573-833a-4dbb-8389-b325b521cd4a"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:35.938196055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 80 len 36864 data: 5 7 8 8 8 4 3 4 4 02/30 Read block 343 len 16384 03/30 Read block 165 len 12288 04/30 Write block 67 len 32768 data: 3 3 3 4 4 4 4 4 05/30 Flush 06/30 Write block 40 len 40960 data: 5 4 4 6 6 7 5 4 4 3 07/30 Write block 542 len 28672 data: 4 4 4 4 3 3 3 08/30 Read block 84 len 40960 09/30 Read block 107 len 36864 10/30 Read block 39 len 36864 11/30 Write block 562 len 12288 data: 3 3 3 12/30 Read block 310 len 28672 13/30 Flush 14/30 Write block 88 len 28672 data: 5 6 6 6 7 6 6 15/30 Read block 41 len 16384 16/30 Read block 180 len 20480 17/30 Flush 18/30 Write block 402 len 24576 data: 4 4 4 4 2 3 19/30 Read block 142 len 4096 20/30 Write block 68 len 24576 data: 4 4 5 5 5 5 21/30 Read block 130 len 40960 22/30 Write block 314 len 36864 data: 3 3 3 3 3 5 6 9 8 23/30 Read block 29 len 28672 24/30 Write block 534 len 24576 data: 3 3 5 4 3 4 25/30 Read block 200 len 4096 26/30 Read block 385 len 8192 27/30 Write block 41 len 36864 data: 5 5 7 7 8 6 5 5 4 28/30 Read block 329 len 32768 29/30 Write block 95 len 8192 data: 5 3 30/30 Read block 566 len 32768 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:35.95985646Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1376} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 78 78 78 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 48 48 48 80 80 80 T T T 3 060-079 48 48 48 79 79 79 T T T 4 080-099 48 48 48 81 81 81 T T T 5 100-119 44 44 44 73 73 73 F F F 6 120-139 45 45 45 76 76 76 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 47 47 47 78 78 78 F F F 10 200-219 47 47 47 78 78 78 F F F 11 220-239 44 44 44 75 75 75 F F F 12 240-259 47 47 47 78 78 78 F F F 13 260-279 44 44 44 73 73 73 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 35 35 35 59 59 59 T T T 16 320-339 43 43 43 72 72 72 T T T 17 340-359 45 45 45 76 76 76 F F F 18 360-379 47 47 47 78 78 78 F F F 19 380-399 47 47 47 78 78 78 F F F 20 400-419 24 24 24 43 43 43 T T T 21 420-439 43 43 43 72 72 72 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 47 47 47 78 78 78 F F F 24 480-499 47 47 47 78 78 78 F F F 25 500-519 44 44 44 73 73 73 F F F 26 520-539 45 45 45 76 76 76 T T T 27 540-559 48 48 48 80 80 80 F F F 28 560-579 48 48 48 80 80 80 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 48, Max flush: 81 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 78 78 78 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 49 49 49 82 82 82 F F F 4 080-099 49 49 49 82 82 82 F F F 5 100-119 44 44 44 73 73 73 F F F 6 120-139 45 45 45 76 76 76 F F F 7 140-159 37 37 37 62 62 62 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 47 47 47 78 78 78 F F F 10 200-219 47 47 47 78 78 78 F F F 11 220-239 44 44 44 75 75 75 F F F 12 240-259 47 47 47 78 78 78 F F F 13 260-279 44 44 44 73 73 73 F F F 14 280-299 25 25 25 45 45 45 F F F 15 300-319 49 49 49 82 82 82 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 45 45 45 76 76 76 F F F 18 360-379 47 47 47 78 78 78 F F F 19 380-399 47 47 47 78 78 78 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 43 43 43 72 72 72 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 47 47 47 78 78 78 F F F 24 480-499 47 47 47 78 78 78 F F F 25 500-519 44 44 44 73 73 73 F F F 26 520-539 49 49 49 82 82 82 F F F 27 540-559 48 48 48 80 80 80 F F F 28 560-579 48 48 48 80 80 80 F F F 29 580-599 31 31 31 53 53 53 F F F Max gen: 49, Max flush: 82 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.247028804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:40.290436493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.290830574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.322719292Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.323398858Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.32343426Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.323448037Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.323462264Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384} {"msg":"Crucible 6294edf3-bd97-42fc-9912-042affcd27ad has session id: 91b4d943-bd5d-4e77-abcf-9b25a8ca2a85","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.32401052Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Upstairs opts: Upstairs UUID: 6294edf3-bd97-42fc-9912-042affcd27ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324063718Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Crucible stats registered with UUID: 6294edf3-bd97-42fc-9912-042affcd27ad","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324084813Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:40.324101079Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324580351Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324602846Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384} {"msg":"6294edf3-bd97-42fc-9912-042affcd27ad active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324660862Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324678338Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"0","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324701243Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"0","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.32471482Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"1","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324728757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"1","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324741654Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"2","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324755371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"2","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324768128Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324792443Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324818917Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"ds_connection connected from Ok(127.0.0.1:61091)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.32484717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"ds_connection connected from Ok(127.0.0.1:39922)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324874334Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"ds_connection connected from Ok(127.0.0.1:60403)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.324902098Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.325670524Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"2","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.325702686Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"0","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.32572942Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"1","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.326284335Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"1","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.326325036Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"2","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.326355598Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"0","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.3269216Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.326947984Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[0]R flush_numbers[0..12]: [78, 73, 82, 82, 82, 73, 76, 62, 78, 78, 78, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.326975008Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[0]R generation[0..12]: [47, 44, 49, 49, 49, 44, 45, 37, 47, 47, 47, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.326990055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327008941Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[1]R flush_numbers[0..12]: [78, 73, 82, 82, 82, 73, 76, 62, 78, 78, 78, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327023447Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[1]R generation[0..12]: [47, 44, 49, 49, 49, 44, 45, 37, 47, 47, 47, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327036734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327050401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[2]R flush_numbers[0..12]: [78, 73, 82, 82, 82, 73, 76, 62, 78, 78, 78, 75]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327063558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[2]R generation[0..12]: [47, 44, 49, 49, 49, 44, 45, 37, 47, 47, 47, 44]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327076785Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327090472Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327103049Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327116016Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327128983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327141991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"":"downstairs","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327155217Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327166865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"0","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:20:40.327179952Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"1","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327209735Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"client":"2","":"downstairs","session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327236069Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327259604Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"6294edf3-bd97-42fc-9912-042affcd27ad is now active with session: 91b4d943-bd5d-4e77-abcf-9b25a8ca2a85","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327284558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327308683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384,"session_id":"91b4d943-bd5d-4e77-abcf-9b25a8ca2a85"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:40.327333987Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 71 len 8192 data: 6 6 02/30 Write block 472 len 24576 data: 6 4 3 3 3 2 03/30 Flush 04/30 Flush 05/30 Write block 472 len 16384 data: 7 5 4 4 06/30 Read block 218 len 28672 07/30 Write block 500 len 36864 data: 6 6 5 4 4 2 3 3 4 08/30 Flush 09/30 Write block 287 len 24576 data: 2 2 3 3 3 3 10/30 Read block 360 len 32768 11/30 Read block 144 len 36864 12/30 Write block 424 len 36864 data: 5 6 6 5 5 5 4 5 7 13/30 Write block 222 len 36864 data: 6 8 6 6 6 6 6 5 5 14/30 Write block 479 len 36864 data: 4 4 5 5 5 5 5 5 7 15/30 Write block 562 len 12288 data: 4 4 4 16/30 Write block 385 len 36864 data: 6 4 3 3 4 4 3 4 6 17/30 Read block 160 len 8192 18/30 Write block 267 len 12288 data: 7 8 7 19/30 Write block 353 len 40960 data: 8 7 7 8 7 6 4 4 4 3 20/30 Write block 145 len 28672 data: 7 8 7 7 7 5 4 21/30 Read block 123 len 36864 22/30 Write block 562 len 8192 data: 5 5 23/30 Read block 255 len 40960 24/30 Read block 485 len 36864 25/30 Write block 233 len 12288 data: 3 5 8 26/30 Flush 27/30 Write block 578 len 24576 data: 4 5 5 6 7 8 28/30 Read block 445 len 24576 29/30 Write block 526 len 20480 data: 7 5 6 7 6 30/30 Write block 182 len 16384 data: 6 5 4 5 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:40.3464817Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1384} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 78 78 78 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 50 50 50 83 83 83 F F F 4 080-099 49 49 49 82 82 82 F F F 5 100-119 44 44 44 73 73 73 F F F 6 120-139 45 45 45 76 76 76 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 47 47 47 78 78 78 T T T 10 200-219 47 47 47 78 78 78 F F F 11 220-239 50 50 50 86 86 86 F F F 12 240-259 47 47 47 78 78 78 F F F 13 260-279 50 50 50 86 86 86 F F F 14 280-299 50 50 50 86 86 86 F F F 15 300-319 49 49 49 82 82 82 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 50 50 50 86 86 86 F F F 18 360-379 50 50 50 86 86 86 F F F 19 380-399 50 50 50 86 86 86 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 50 50 50 86 86 86 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 50 50 50 86 86 86 F F F 24 480-499 50 50 50 86 86 86 F F F 25 500-519 50 50 50 85 85 85 F F F 26 520-539 49 49 49 82 82 82 T T T 27 540-559 48 48 48 80 80 80 F F F 28 560-579 50 50 50 86 86 86 T T T 29 580-599 31 31 31 53 53 53 T T T Max gen: 50, Max flush: 86 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 78 78 78 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 50 50 50 83 83 83 F F F 4 080-099 49 49 49 82 82 82 F F F 5 100-119 44 44 44 73 73 73 F F F 6 120-139 45 45 45 76 76 76 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 47 47 47 78 78 78 F F F 11 220-239 50 50 50 86 86 86 F F F 12 240-259 47 47 47 78 78 78 F F F 13 260-279 50 50 50 86 86 86 F F F 14 280-299 50 50 50 86 86 86 F F F 15 300-319 49 49 49 82 82 82 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 50 50 50 86 86 86 F F F 18 360-379 50 50 50 86 86 86 F F F 19 380-399 50 50 50 86 86 86 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 50 50 50 86 86 86 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 50 50 50 86 86 86 F F F 24 480-499 50 50 50 86 86 86 F F F 25 500-519 50 50 50 85 85 85 F F F 26 520-539 51 51 51 87 87 87 F F F 27 540-559 48 48 48 80 80 80 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 51 51 51 87 87 87 F F F Max gen: 51, Max flush: 87 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.615923923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.662131738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.662885507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.662937455Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.662963719Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.662988734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392} {"msg":"Crucible 0dd8ef40-9f69-4e61-bb12-b58f14905d20 has session id: 9b7fab37-0aa7-46ee-af8d-540a7f69fe71","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.663596906Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Upstairs opts: Upstairs UUID: 0dd8ef40-9f69-4e61-bb12-b58f14905d20, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.663654533Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Crucible stats registered with UUID: 0dd8ef40-9f69-4e61-bb12-b58f14905d20","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.663675898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:44.663689375Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664080006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664104291Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392} {"msg":"0dd8ef40-9f69-4e61-bb12-b58f14905d20 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664186522Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664204608Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"0","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664227763Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"0","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.66424179Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"1","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664259376Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"1","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664272663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"2","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.66428645Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"2","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664299667Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664323151Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664348196Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"ds_connection connected from Ok(127.0.0.1:50475)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.66437343Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"ds_connection connected from Ok(127.0.0.1:57116)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664400594Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"ds_connection connected from Ok(127.0.0.1:60714)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.664508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.665177948Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"0","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.665207731Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"2","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.665232616Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"1","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.665895046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"1","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.665992413Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"0","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666014159Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"2","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666562814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666588778Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[0]R flush_numbers[0..12]: [78, 73, 82, 83, 82, 73, 76, 86, 78, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666630059Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[0]R generation[0..12]: [47, 44, 49, 50, 49, 44, 45, 50, 47, 51, 47, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666647315Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666661382Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[1]R flush_numbers[0..12]: [78, 73, 82, 83, 82, 73, 76, 86, 78, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666674779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[1]R generation[0..12]: [47, 44, 49, 50, 49, 44, 45, 50, 47, 51, 47, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666688176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666702053Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[2]R flush_numbers[0..12]: [78, 73, 82, 83, 82, 73, 76, 86, 78, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.66671545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[2]R generation[0..12]: [47, 44, 49, 50, 49, 44, 45, 50, 47, 51, 47, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666728987Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666742543Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666755071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666767828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666780515Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666797471Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"":"downstairs","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"No downstairs reconciliation requiredWait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666858417Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666873204Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"0","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666887441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"1","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666900818Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"client":"2","":"downstairs","session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666913965Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666926022Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"0dd8ef40-9f69-4e61-bb12-b58f14905d20 is now active with session: 9b7fab37-0aa7-46ee-af8d-540a7f69fe71","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666938479Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666950167Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392,"session_id":"9b7fab37-0aa7-46ee-af8d-540a7f69fe71"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:44.666961994Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 462 len 20480 data: 3 5 6 5 6 02/30 Read block 122 len 24576 03/30 Read block 177 len 24576 04/30 Write block 278 len 12288 data: 3 3 3 05/30 Write block 79 len 40960 data: 4 6 8 9 9 9 5 4 5 6 06/30 Read block 108 len 36864 07/30 Flush 08/30 Write block 128 len 20480 data: 3 3 4 3 3 09/30 Flush 10/30 Write block 67 len 36864 data: 4 5 5 6 7 7 6 5 2 11/30 Read block 374 len 28672 12/30 Read block 561 len 8192 13/30 Read block 314 len 4096 14/30 Read block 148 len 16384 15/30 Read block 8 len 36864 16/30 Read block 274 len 12288 17/30 Write block 390 len 16384 data: 5 4 5 7 18/30 Read block 227 len 12288 19/30 Read block 15 len 24576 20/30 Read block 344 len 12288 21/30 Flush 22/30 Write block 246 len 20480 data: 3 3 4 5 4 23/30 Write block 433 len 28672 data: 8 9 6 5 5 5 4 24/30 Read block 465 len 36864 25/30 Read block 540 len 32768 26/30 Read block 264 len 40960 27/30 Write block 494 len 12288 data: 4 4 3 28/30 Read block 226 len 16384 29/30 Read block 60 len 24576 30/30 Read block 484 len 16384 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 9 8 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:44.687779529Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1392} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 78 78 78 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 52 52 52 90 90 90 F F F 4 080-099 52 52 52 88 88 88 F F F 5 100-119 44 44 44 73 73 73 F F F 6 120-139 52 52 52 89 89 89 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 47 47 47 78 78 78 F F F 11 220-239 50 50 50 86 86 86 F F F 12 240-259 47 47 47 78 78 78 T T T 13 260-279 52 52 52 88 88 88 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 49 49 49 82 82 82 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 50 50 50 86 86 86 F F F 18 360-379 50 50 50 86 86 86 F F F 19 380-399 52 52 52 90 90 90 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 50 50 50 86 86 86 T T T 22 440-459 44 44 44 73 73 73 F F F 23 460-479 52 52 52 88 88 88 F F F 24 480-499 50 50 50 86 86 86 T T T 25 500-519 50 50 50 85 85 85 F F F 26 520-539 51 51 51 87 87 87 F F F 27 540-559 48 48 48 80 80 80 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 51 51 51 87 87 87 F F F Max gen: 52, Max flush: 90 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 47 47 47 78 78 78 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 49 49 49 82 82 82 F F F 3 060-079 52 52 52 90 90 90 F F F 4 080-099 52 52 52 88 88 88 F F F 5 100-119 44 44 44 73 73 73 F F F 6 120-139 52 52 52 89 89 89 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 47 47 47 78 78 78 F F F 11 220-239 50 50 50 86 86 86 F F F 12 240-259 53 53 53 91 91 91 F F F 13 260-279 52 52 52 88 88 88 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 49 49 49 82 82 82 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 50 50 50 86 86 86 F F F 18 360-379 50 50 50 86 86 86 F F F 19 380-399 52 52 52 90 90 90 F F F 20 400-419 49 49 49 82 82 82 F F F 21 420-439 53 53 53 91 91 91 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 52 52 52 88 88 88 F F F 24 480-499 53 53 53 91 91 91 F F F 25 500-519 50 50 50 85 85 85 F F F 26 520-539 51 51 51 87 87 87 F F F 27 540-559 48 48 48 80 80 80 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 51 51 51 87 87 87 F F F Max gen: 53, Max flush: 91 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.926698574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.971435192Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.972055002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.972095553Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.972110839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.972125306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400} {"msg":"Crucible a17800da-b51c-4436-a608-18a95a4a9750 has session id: b23b2567-8be8-491d-a34a-17166d122394","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.972649207Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Upstairs opts: Upstairs UUID: a17800da-b51c-4436-a608-18a95a4a9750, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.972741176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Crucible stats registered with UUID: a17800da-b51c-4436-a608-18a95a4a9750","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.972774889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:48.972795934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973156162Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973181547Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400} {"msg":"a17800da-b51c-4436-a608-18a95a4a9750 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973263038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973281384Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"0","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973304359Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"0","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973317966Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973335702Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973359766Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"2","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973385761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"2","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973403596Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973417283Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973441548Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"ds_connection connected from Ok(127.0.0.1:48692)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973574478Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"ds_connection connected from Ok(127.0.0.1:51304)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973700659Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:48.973735941Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:48.973765914Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:48.973796108Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.973841247Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.974184849Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"2","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.974211663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"0","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.97488805Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"2","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.974928021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"0","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.975432057Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:48.975513069Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.975184651Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"ds_connection connected from Ok(127.0.0.1:60788)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.975356582Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.976441916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.976950991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[0]R flush_numbers[0..12]: [78, 73, 82, 90, 88, 73, 89, 86, 78, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977521752Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[0]R generation[0..12]: [47, 44, 49, 52, 52, 44, 52, 50, 47, 51, 47, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977550585Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977569371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[1]R flush_numbers[0..12]: [78, 73, 82, 90, 88, 73, 89, 86, 78, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977594265Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[1]R generation[0..12]: [47, 44, 49, 52, 52, 44, 52, 50, 47, 51, 47, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977611881Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977628908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[2]R flush_numbers[0..12]: [78, 73, 82, 90, 88, 73, 89, 86, 78, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977645814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[2]R generation[0..12]: [47, 44, 49, 52, 52, 44, 52, 50, 47, 51, 47, 50]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.97766248Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977679636Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977695682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977711959Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977727745Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977743362Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"":"downstairs","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977759558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977774215Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"0","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977790441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"1","":"downstairs","session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977806367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"client":"2","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977833591Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977849158Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"a17800da-b51c-4436-a608-18a95a4a9750 is now active with session: b23b2567-8be8-491d-a34a-17166d122394","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977864934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977879631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400,"session_id":"b23b2567-8be8-491d-a34a-17166d122394"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:20:58.977894557Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 94 len 36864 data: 7 6 4 4 3 2 3 5 5 02/30 Write block 544 len 24576 data: 5 5 4 4 4 4 03/30 Read block 263 len 12288 04/30 Write block 484 len 20480 data: 6 6 6 8 8 05/30 Read block 272 len 32768 06/30 Write block 402 len 16384 data: 5 5 5 5 07/30 Write block 112 len 36864 data: 4 4 3 5 6 5 4 2 3 08/30 Write block 225 len 28672 data: 7 7 7 7 6 6 6 09/30 Read block 27 len 32768 10/30 Write block 135 len 8192 data: 3 4 11/30 Read block 270 len 8192 12/30 Read block 341 len 40960 13/30 Read block 51 len 4096 14/30 Write block 222 len 24576 data: 7 9 7 8 8 8 15/30 Read block 398 len 24576 16/30 Flush 17/30 Write block 347 len 12288 data: 4 5 7 18/30 Read block 515 len 32768 19/30 Write block 8 len 36864 data: 4 4 2 2 3 3 3 3 4 20/30 Write block 513 len 36864 data: 7 7 7 8 7 7 9 8 8 21/30 Flush 22/30 Write block 317 len 8192 data: 4 4 23/30 Read block 532 len 4096 24/30 Write block 484 len 28672 data: 7 7 7 9 9 5 5 25/30 Write block 278 len 4096 data: 4 26/30 Write block 346 len 16384 data: 3 5 6 8 27/30 Write block 53 len 28672 data: 3 2 2 3 3 3 3 28/30 Read block 8 len 20480 29/30 Write block 363 len 12288 data: 4 6 6 30/30 Read block 469 len 16384 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 9 7 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:20:58.997251983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1400} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 93 93 93 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 49 49 49 82 82 82 T T T 3 060-079 52 52 52 90 90 90 F F F 4 080-099 54 54 54 92 92 92 F F F 5 100-119 54 54 54 92 92 92 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 47 47 47 78 78 78 F F F 11 220-239 54 54 54 92 92 92 F F F 12 240-259 53 53 53 91 91 91 F F F 13 260-279 52 52 52 88 88 88 T T T 14 280-299 52 52 52 88 88 88 F F F 15 300-319 49 49 49 82 82 82 T T T 16 320-339 49 49 49 82 82 82 F F F 17 340-359 54 54 54 93 93 93 T T T 18 360-379 50 50 50 86 86 86 T T T 19 380-399 52 52 52 90 90 90 F F F 20 400-419 54 54 54 92 92 92 F F F 21 420-439 53 53 53 91 91 91 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 52 52 52 88 88 88 F F F 24 480-499 54 54 54 92 92 92 T T T 25 500-519 54 54 54 93 93 93 F F F 26 520-539 54 54 54 93 93 93 F F F 27 540-559 54 54 54 92 92 92 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 51 51 51 87 87 87 F F F Max gen: 54, Max flush: 93 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 93 93 93 F F F 1 020-039 44 44 44 73 73 73 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 52 52 52 90 90 90 F F F 4 080-099 54 54 54 92 92 92 F F F 5 100-119 54 54 54 92 92 92 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 47 47 47 78 78 78 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 47 47 47 78 78 78 F F F 11 220-239 54 54 54 92 92 92 F F F 12 240-259 53 53 53 91 91 91 F F F 13 260-279 55 55 55 94 94 94 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 55 55 55 94 94 94 F F F 16 320-339 49 49 49 82 82 82 F F F 17 340-359 55 55 55 94 94 94 F F F 18 360-379 55 55 55 94 94 94 F F F 19 380-399 52 52 52 90 90 90 F F F 20 400-419 54 54 54 92 92 92 F F F 21 420-439 53 53 53 91 91 91 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 52 52 52 88 88 88 F F F 24 480-499 55 55 55 94 94 94 F F F 25 500-519 54 54 54 93 93 93 F F F 26 520-539 54 54 54 93 93 93 F F F 27 540-559 54 54 54 92 92 92 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 51 51 51 87 87 87 F F F Max gen: 55, Max flush: 94 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.245752488Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.291157444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.291791971Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.291852337Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.291875302Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.291899036Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408} {"msg":"Crucible 1b30e39d-31b7-4307-b2d7-6fe5acbe95b8 has session id: ef83f9cf-e693-4e62-b5ca-6ad99c651d01","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.29254609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Upstairs opts: Upstairs UUID: 1b30e39d-31b7-4307-b2d7-6fe5acbe95b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.29258909Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Crucible stats registered with UUID: 1b30e39d-31b7-4307-b2d7-6fe5acbe95b8","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.292606346Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:03.292619903Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293116281Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293142745Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408} {"msg":"1b30e39d-31b7-4307-b2d7-6fe5acbe95b8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293201202Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293219517Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"0","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293241512Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"0","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293255029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"1","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293268976Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"1","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293286462Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"2","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293358626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"2","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293388789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293416233Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293444837Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"ds_connection connected from Ok(127.0.0.1:37599)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.2934716Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"ds_connection connected from Ok(127.0.0.1:59542)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293497494Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"ds_connection connected from Ok(127.0.0.1:60901)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.293524039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.294081372Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"2","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.294110696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"0","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.294249834Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"1","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.294738613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"0","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.294766507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"2","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.294894588Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"1","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295288899Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295397074Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[0]R flush_numbers[0..12]: [93, 73, 94, 90, 92, 92, 92, 86, 78, 87, 78, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295463879Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[0]R generation[0..12]: [54, 44, 55, 52, 54, 54, 54, 50, 47, 51, 47, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295490853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.29550573Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[1]R flush_numbers[0..12]: [93, 73, 94, 90, 92, 92, 92, 86, 78, 87, 78, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295519876Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[1]R generation[0..12]: [54, 44, 55, 52, 54, 54, 54, 50, 47, 51, 47, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295533304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.29554699Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[2]R flush_numbers[0..12]: [93, 73, 94, 90, 92, 92, 92, 86, 78, 87, 78, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295560267Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[2]R generation[0..12]: [54, 44, 55, 52, 54, 54, 54, 50, 47, 51, 47, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295573664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295587371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295600198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295612915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295625373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295642469Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"":"downstairs","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295656106Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295675921Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"0","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295689348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"1","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295702335Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"client":"2","":"downstairs","session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295715332Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295727349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"1b30e39d-31b7-4307-b2d7-6fe5acbe95b8 is now active with session: ef83f9cf-e693-4e62-b5ca-6ad99c651d01","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295739947Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295751584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408,"session_id":"ef83f9cf-e693-4e62-b5ca-6ad99c651d01"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:03.295763441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 331 len 24576 data: 3 3 4 4 6 6 02/30 Write block 374 len 4096 data: 6 03/30 Read block 470 len 36864 04/30 Read block 349 len 32768 05/30 Read block 231 len 36864 06/30 Read block 41 len 24576 07/30 Read block 85 len 4096 08/30 Read block 425 len 16384 09/30 Flush 10/30 Write block 495 len 4096 data: 5 11/30 Read block 294 len 28672 12/30 Read block 531 len 40960 13/30 Read block 534 len 16384 14/30 Read block 55 len 8192 15/30 Write block 241 len 12288 data: 8 8 7 16/30 Read block 272 len 32768 17/30 Read block 7 len 40960 18/30 Flush 19/30 Write block 354 len 24576 data: 8 8 9 8 7 5 20/30 Read block 201 len 4096 21/30 Write block 373 len 4096 data: 6 22/30 Write block 28 len 20480 data: 6 5 5 5 5 23/30 Write block 494 len 40960 data: 5 6 4 4 5 5 7 7 6 5 24/30 Write block 325 len 4096 data: 5 25/30 Write block 168 len 16384 data: 3 4 3 5 26/30 Write block 304 len 4096 data: 5 27/30 Write block 429 len 16384 data: 6 5 6 8 28/30 Read block 439 len 28672 29/30 Write block 216 len 16384 data: 5 6 7 6 30/30 Read block 63 len 4096 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:03.313631554Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1408} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 93 93 93 F F F 1 020-039 44 44 44 73 73 73 T T T 2 040-059 55 55 55 94 94 94 F F F 3 060-079 52 52 52 90 90 90 F F F 4 080-099 54 54 54 92 92 92 F F F 5 100-119 54 54 54 92 92 92 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 47 47 47 78 78 78 T T T 9 180-199 51 51 51 87 87 87 F F F 10 200-219 47 47 47 78 78 78 T T T 11 220-239 54 54 54 92 92 92 F F F 12 240-259 56 56 56 96 96 96 F F F 13 260-279 55 55 55 94 94 94 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 55 55 55 94 94 94 T T T 16 320-339 56 56 56 95 95 95 T T T 17 340-359 55 55 55 94 94 94 T T T 18 360-379 56 56 56 95 95 95 T T T 19 380-399 52 52 52 90 90 90 F F F 20 400-419 54 54 54 92 92 92 F F F 21 420-439 53 53 53 91 91 91 T T T 22 440-459 44 44 44 73 73 73 F F F 23 460-479 52 52 52 88 88 88 F F F 24 480-499 56 56 56 96 96 96 T T T 25 500-519 54 54 54 93 93 93 T T T 26 520-539 54 54 54 93 93 93 F F F 27 540-559 54 54 54 92 92 92 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 51 51 51 87 87 87 F F F Max gen: 56, Max flush: 96 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 93 93 93 F F F 1 020-039 57 57 57 97 97 97 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 52 52 52 90 90 90 F F F 4 080-099 54 54 54 92 92 92 F F F 5 100-119 54 54 54 92 92 92 F F F 6 120-139 54 54 54 92 92 92 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 57 57 57 97 97 97 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 57 57 57 97 97 97 F F F 11 220-239 54 54 54 92 92 92 F F F 12 240-259 56 56 56 96 96 96 F F F 13 260-279 55 55 55 94 94 94 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 97 97 97 F F F 16 320-339 57 57 57 97 97 97 F F F 17 340-359 57 57 57 97 97 97 F F F 18 360-379 57 57 57 97 97 97 F F F 19 380-399 52 52 52 90 90 90 F F F 20 400-419 54 54 54 92 92 92 F F F 21 420-439 57 57 57 97 97 97 F F F 22 440-459 44 44 44 73 73 73 F F F 23 460-479 52 52 52 88 88 88 F F F 24 480-499 57 57 57 97 97 97 F F F 25 500-519 57 57 57 97 97 97 F F F 26 520-539 54 54 54 93 93 93 F F F 27 540-559 54 54 54 92 92 92 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 51 51 51 87 87 87 F F F Max gen: 57, Max flush: 97 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.578706286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.624225717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.624958151Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.62500388Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.625018957Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.625033534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416} {"msg":"Crucible ce80799b-7fb1-47fe-a6f6-30b70bc8754f has session id: 849a9436-9055-480c-84bc-d10dff7c5bf4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.625738244Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Upstairs opts: Upstairs UUID: ce80799b-7fb1-47fe-a6f6-30b70bc8754f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.625771446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Crucible stats registered with UUID: ce80799b-7fb1-47fe-a6f6-30b70bc8754f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.625786893Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:07.6257996Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.62619819Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626222444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416} {"msg":"ce80799b-7fb1-47fe-a6f6-30b70bc8754f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626289309Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626308445Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"0","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626331289Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"0","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626345206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"1","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626364032Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"1","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626396835Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"2","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626423469Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"2","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626447853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626471748Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626495063Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:47553)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626519387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43346)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626602688Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:37758)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.626743696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.627306609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"0","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.627334752Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"1","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.627355997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"2","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.627928988Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"1","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.627957631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"0","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.627972988Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"2","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628481533Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628503897Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[0]R flush_numbers[0..12]: [93, 97, 94, 90, 92, 92, 92, 86, 97, 87, 97, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628577181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[0]R generation[0..12]: [54, 57, 55, 52, 54, 54, 54, 50, 57, 51, 57, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628596007Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628610353Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[1]R flush_numbers[0..12]: [93, 97, 94, 90, 92, 92, 92, 86, 97, 87, 97, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.62862392Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[1]R generation[0..12]: [54, 57, 55, 52, 54, 54, 54, 50, 57, 51, 57, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628637277Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628650904Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[2]R flush_numbers[0..12]: [93, 97, 94, 90, 92, 92, 92, 86, 97, 87, 97, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628664471Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[2]R generation[0..12]: [54, 57, 55, 52, 54, 54, 54, 50, 57, 51, 57, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628677898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628695434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628708651Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628721478Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628734175Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628746802Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"":"downstairs","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.62875999Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628771847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"0","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:21:07.628785134Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"1","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628804489Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"client":"2","":"downstairs","session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628817686Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628829974Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"ce80799b-7fb1-47fe-a6f6-30b70bc8754f is now active with session: 849a9436-9055-480c-84bc-d10dff7c5bf4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628842381Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628854068Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416,"session_id":"849a9436-9055-480c-84bc-d10dff7c5bf4"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:07.628866086Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 517 len 16384 data: 8 8 10 9 02/30 Write block 502 len 20480 data: 7 6 5 3 4 03/30 Read block 251 len 28672 04/30 Write block 273 len 4096 data: 5 05/30 Read block 504 len 28672 06/30 Read block 546 len 12288 07/30 Flush 08/30 Write block 444 len 12288 data: 4 4 3 09/30 Read block 22 len 8192 10/30 Write block 384 len 36864 data: 7 7 5 4 4 5 6 5 6 11/30 Write block 61 len 20480 data: 4 4 3 3 4 12/30 Flush 13/30 Write block 253 len 8192 data: 3 3 14/30 Write block 105 len 20480 data: 4 5 6 7 5 15/30 Read block 243 len 4096 16/30 Read block 170 len 20480 17/30 Read block 376 len 4096 18/30 Flush 19/30 Write block 258 len 16384 data: 2 3 4 4 20/30 Write block 113 len 32768 data: 5 4 6 7 6 5 3 4 21/30 Read block 356 len 8192 22/30 Write block 427 len 16384 data: 6 6 7 6 23/30 Read block 165 len 12288 24/30 Read block 246 len 40960 25/30 Read block 189 len 16384 26/30 Write block 203 len 32768 data: 6 5 5 4 5 5 6 4 27/30 Write block 599 len 4096 data: 2 28/30 Read block 593 len 20480 29/30 Read block 136 len 24576 30/30 Write block 384 len 28672 data: 8 8 6 5 5 6 7 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:07.647606551Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1416} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 93 93 93 F F F 1 020-039 57 57 57 97 97 97 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 58 58 58 99 99 99 F F F 4 080-099 54 54 54 92 92 92 F F F 5 100-119 58 58 58 100 100 100 T T T 6 120-139 54 54 54 92 92 92 T T T 7 140-159 50 50 50 86 86 86 F F F 8 160-179 57 57 57 97 97 97 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 57 57 57 97 97 97 T T T 11 220-239 54 54 54 92 92 92 F F F 12 240-259 58 58 58 100 100 100 T T T 13 260-279 58 58 58 98 98 98 T T T 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 97 97 97 F F F 16 320-339 57 57 57 97 97 97 F F F 17 340-359 57 57 57 97 97 97 F F F 18 360-379 57 57 57 97 97 97 F F F 19 380-399 58 58 58 99 99 99 T T T 20 400-419 54 54 54 92 92 92 F F F 21 420-439 57 57 57 97 97 97 T T T 22 440-459 58 58 58 99 99 99 F F F 23 460-479 52 52 52 88 88 88 F F F 24 480-499 57 57 57 97 97 97 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 58 58 58 98 98 98 F F F 27 540-559 54 54 54 92 92 92 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 51 51 51 87 87 87 T T T Max gen: 58, Max flush: 100 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 93 93 93 F F F 1 020-039 57 57 57 97 97 97 F F F 2 040-059 55 55 55 94 94 94 F F F 3 060-079 58 58 58 99 99 99 F F F 4 080-099 54 54 54 92 92 92 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 59 59 59 101 101 101 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 57 57 57 97 97 97 F F F 9 180-199 51 51 51 87 87 87 F F F 10 200-219 59 59 59 101 101 101 F F F 11 220-239 54 54 54 92 92 92 F F F 12 240-259 59 59 59 101 101 101 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 97 97 97 F F F 16 320-339 57 57 57 97 97 97 F F F 17 340-359 57 57 57 97 97 97 F F F 18 360-379 57 57 57 97 97 97 F F F 19 380-399 59 59 59 101 101 101 F F F 20 400-419 54 54 54 92 92 92 F F F 21 420-439 59 59 59 101 101 101 F F F 22 440-459 58 58 58 99 99 99 F F F 23 460-479 52 52 52 88 88 88 F F F 24 480-499 57 57 57 97 97 97 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 58 58 58 98 98 98 F F F 27 540-559 54 54 54 92 92 92 F F F 28 560-579 51 51 51 87 87 87 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 59, Max flush: 101 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.908819825Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.953896327Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.954669991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.954712212Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.954728038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.954746374Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424} {"msg":"Crucible 30ec0f0c-30e8-46ff-bf9b-f2631ed2268a has session id: c1891f75-57de-403b-af85-3f71b95d34fa","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.955350277Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Upstairs opts: Upstairs UUID: 30ec0f0c-30e8-46ff-bf9b-f2631ed2268a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.955399706Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Crucible stats registered with UUID: 30ec0f0c-30e8-46ff-bf9b-f2631ed2268a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.955423521Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:11.955444496Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.955886146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.95591288Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424} {"msg":"30ec0f0c-30e8-46ff-bf9b-f2631ed2268a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.955980775Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.9559988Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"0","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956022105Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"0","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956035882Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"1","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956049949Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"1","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956064305Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"2","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.95608854Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"2","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956113504Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956138729Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956164333Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:58687)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956193296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:60502)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956256492Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"ds_connection connected from Ok(127.0.0.1:44982)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956286445Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.956978218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"2","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.957007682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"0","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.957035396Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"1","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.957719121Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"1","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.957757772Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"2","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.957785696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"0","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.9583399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958374692Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[0]R flush_numbers[0..12]: [93, 97, 94, 99, 92, 101, 101, 86, 97, 87, 101, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.95842963Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[0]R generation[0..12]: [54, 57, 55, 58, 54, 59, 59, 50, 57, 51, 59, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958457903Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.95847367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[1]R flush_numbers[0..12]: [93, 97, 94, 99, 92, 101, 101, 86, 97, 87, 101, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958491776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[1]R generation[0..12]: [54, 57, 55, 58, 54, 59, 59, 50, 57, 51, 59, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958506383Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958520219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[2]R flush_numbers[0..12]: [93, 97, 94, 99, 92, 101, 101, 86, 97, 87, 101, 92]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958533516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[2]R generation[0..12]: [54, 57, 55, 58, 54, 59, 59, 50, 57, 51, 59, 54]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958546663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.95856015Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958572937Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958585944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":Wait for a query_work_queue command to finish before sending IO 1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958654429Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958668746Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"":"downstairs","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958681643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.95869327Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"0","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958706287Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"1","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958719214Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"client":"2","":"downstairs","session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958732151Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958747948Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"30ec0f0c-30e8-46ff-bf9b-f2631ed2268a is now active with session: c1891f75-57de-403b-af85-3f71b95d34fa","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958760715Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.958772332Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424,"session_id":"c1891f75-57de-403b-af85-3f71b95d34fa"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:11.9587842Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 438 len 32768 data: 6 5 4 5 4 5 5 5 02/30 Write block 470 len 28672 data: 6 7 8 6 5 5 4 03/30 Write block 43 len 20480 data: 8 8 9 7 6 04/30 Write block 560 len 16384 data: 4 3 6 6 05/30 Write block 89 len 24576 data: 7 7 7 8 7 8 06/30 Flush 07/30 Write block 68 len 16384 data: 6 6 7 8 08/30 Write block 228 len 24576 data: 8 7 7 7 4 4 09/30 Read block 175 len 24576 10/30 Flush 11/30 Write block 2 len 12288 data: 4 5 5 12/30 Read block 24 len 20480 13/30 Write block 209 len 24576 data: 7 5 5 5 5 4 14/30 Read block 185 len 36864 15/30 Write block 522 len 24576 data: 9 8 8 8 8 6 16/30 Read block 560 len 12288 17/30 Write block 465 len 40960 data: 6 7 6 7 6 7 8 9 7 6 18/30 Read block 193 len 24576 19/30 Read block 530 len 40960 20/30 Read block 588 len 24576 21/30 Write block 542 len 40960 data: 5 5 6 6 5 5 5 5 5 6 22/30 Read block 389 len 36864 23/30 Read block 467 len 16384 24/30 Read block 555 len 12288 25/30 Flush 26/30 Write block 176 len 36864 data: 6 6 6 7 8 8 7 6 5 27/30 Write block 22 len 24576 data: 5 4 4 4 6 5 28/30 Read block 111 len 28672 29/30 Read block 262 len 32768 30/30 Read block 77 len 32768 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:11.979232038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1424} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 104 104 104 F F F 1 020-039 57 57 57 97 97 97 T T T 2 040-059 60 60 60 102 102 102 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 59 59 59 101 101 101 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 57 57 57 97 97 97 T T T 9 180-199 51 51 51 87 87 87 T T T 10 200-219 60 60 60 104 104 104 F F F 11 220-239 60 60 60 103 103 103 F F F 12 240-259 59 59 59 101 101 101 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 97 97 97 F F F 16 320-339 57 57 57 97 97 97 F F F 17 340-359 57 57 57 97 97 97 F F F 18 360-379 57 57 57 97 97 97 F F F 19 380-399 59 59 59 101 101 101 F F F 20 400-419 54 54 54 92 92 92 F F F 21 420-439 60 60 60 102 102 102 F F F 22 440-459 60 60 60 102 102 102 F F F 23 460-479 60 60 60 104 104 104 F F F 24 480-499 57 57 57 97 97 97 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 60 60 60 104 104 104 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 60 60 60 102 102 102 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 60, Max flush: 104 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 104 104 104 F F F 1 020-039 61 61 61 105 105 105 F F F 2 040-059 60 60 60 102 102 102 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 60 60 60 102 102 102 F F F 5 100-119 59 59 59 101 101 101 F F F 6 120-139 59 59 59 101 101 101 F F F 7 140-159 50 50 50 86 86 86 F F F 8 160-179 61 61 61 105 105 105 F F F 9 180-199 61 61 61 105 105 105 F F F 10 200-219 60 60 60 104 104 104 F F F 11 220-239 60 60 60 103 103 103 F F F 12 240-259 59 59 59 101 101 101 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 97 97 97 F F F 16 320-339 57 57 57 97 97 97 F F F 17 340-359 57 57 57 97 97 97 F F F 18 360-379 57 57 57 97 97 97 F F F 19 380-399 59 59 59 101 101 101 F F F 20 400-419 54 54 54 92 92 92 F F F 21 420-439 60 60 60 102 102 102 F F F 22 440-459 60 60 60 102 102 102 F F F 23 460-479 60 60 60 104 104 104 F F F 24 480-499 57 57 57 97 97 97 F F F 25 500-519 58 58 58 98 98 98 F F F 26 520-539 60 60 60 104 104 104 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 60 60 60 102 102 102 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 61, Max flush: 105 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.211368038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.256965571Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.257768889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.257821857Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.257841263Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.257856219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432} {"msg":"Crucible c349d42a-a874-4713-adc1-523be3ffca56 has session id: c77ef6bd-74cc-4d07-a4fe-86b9560bb778","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.258457853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Upstairs opts: Upstairs UUID: c349d42a-a874-4713-adc1-523be3ffca56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.258502003Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Crucible stats registered with UUID: c349d42a-a874-4713-adc1-523be3ffca56","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.258520119Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:16.258533066Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.258943513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.258976386Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432} {"msg":"c349d42a-a874-4713-adc1-523be3ffca56 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259028924Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259055578Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"0","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259081032Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"0","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259095579Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"1","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259109696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"1","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259126362Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"2","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259144368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"2","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259169092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259195246Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.2592218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"ds_connection connected from Ok(127.0.0.1:42519)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259247444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"ds_connection connected from Ok(127.0.0.1:41787)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259272599Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"ds_connection connected from Ok(127.0.0.1:48845)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259299613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.259968941Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"2","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.260011341Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"0","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.260160318Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"1","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.260620413Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"2","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.260648767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"0","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.260798213Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"1","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261146294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261231385Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[0]R flush_numbers[0..12]: [104, 105, 102, 103, 102, 101, 101, 86, 105, 105, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261338681Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[0]R generation[0..12]: [60, 61, 60, 60, 60, 59, 59, 50, 61, 61, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261357496Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261371923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[1]R flush_numbers[0..12]: [104, 105, 102, 103, 102, 101, 101, 86, 105, 105, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.26138591Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[1]R generation[0..12]: [60, 61, 60, 60, 60, 59, 59, 50, 61, 61, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261399687Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261413324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[2]R flush_numbers[0..12]: [104, 105, 102, 103, 102, 101, 101, 86, 105, 105, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261426561Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[2]R generation[0..12]: [60, 61, 60, 60, 60, 59, 59, 50, 61, 61, 60, 60]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261440018Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261453575Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261466432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261479189Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:21:16.261491846Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261507912Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"":"downstairs","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261524258Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261536456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"0","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261549743Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"1","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.26156259Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"client":"2","":"downstairs","session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261575707Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261587574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"c349d42a-a874-4713-adc1-523be3ffca56 is now active with session: c77ef6bd-74cc-4d07-a4fe-86b9560bb778","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261600151Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261611779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432,"session_id":"c77ef6bd-74cc-4d07-a4fe-86b9560bb778"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:16.261623566Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 8192 data: 6 6 02/30 Write block 512 len 20480 data: 5 8 8 8 9 03/30 Read block 506 len 8192 04/30 Write block 3 len 40960 data: 6 6 5 5 5 5 5 3 3 4 05/30 Write block 38 len 32768 data: 6 6 6 6 6 9 9 10 06/30 Read block 186 len 24576 07/30 Write block 253 len 28672 data: 4 4 3 3 2 3 4 08/30 Read block 160 len 24576 09/30 Write block 115 len 36864 data: 7 8 7 6 4 5 4 4 4 10/30 Read block 173 len 16384 11/30 Write block 494 len 32768 data: 6 7 5 5 6 6 8 8 12/30 Write block 95 len 36864 data: 7 5 5 4 3 4 6 6 5 13/30 Write block 150 len 4096 data: 6 14/30 Read block 451 len 28672 15/30 Read block 423 len 40960 16/30 Write block 342 len 28672 data: 3 3 3 3 4 6 7 17/30 Write block 353 len 24576 data: 9 9 9 10 9 8 18/30 Read block 106 len 36864 19/30 Read block 300 len 8192 20/30 Read block 84 len 40960 21/30 Read block 301 len 8192 22/30 Read block 404 len 24576 23/30 Write block 399 len 28672 data: 3 3 4 6 6 6 6 24/30 Write block 386 len 12288 data: 7 7 7 25/30 Read block 356 len 24576 26/30 Write block 576 len 8192 data: 5 5 27/30 Write block 233 len 40960 data: 5 6 9 9 8 9 6 7 9 9 28/30 Write block 407 len 20480 data: 4 3 3 3 3 29/30 Write block 162 len 36864 data: 9 9 7 6 6 5 4 5 4 30/30 Read block 296 len 8192 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:16.281118081Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1432} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 104 104 104 T T T 1 020-039 61 61 61 105 105 105 T T T 2 040-059 60 60 60 102 102 102 T T T 3 060-079 60 60 60 103 103 103 F F F 4 080-099 60 60 60 102 102 102 T T T 5 100-119 59 59 59 101 101 101 T T T 6 120-139 59 59 59 101 101 101 T T T 7 140-159 50 50 50 86 86 86 T T T 8 160-179 61 61 61 105 105 105 T T T 9 180-199 61 61 61 105 105 105 F F F 10 200-219 60 60 60 104 104 104 F F F 11 220-239 60 60 60 103 103 103 T T T 12 240-259 59 59 59 101 101 101 T T T 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 97 97 97 F F F 16 320-339 57 57 57 97 97 97 F F F 17 340-359 57 57 57 97 97 97 T T T 18 360-379 57 57 57 97 97 97 F F F 19 380-399 59 59 59 101 101 101 T T T 20 400-419 54 54 54 92 92 92 T T T 21 420-439 60 60 60 102 102 102 F F F 22 440-459 60 60 60 102 102 102 F F F 23 460-479 60 60 60 104 104 104 F F F 24 480-499 57 57 57 97 97 97 T T T 25 500-519 58 58 58 98 98 98 T T T 26 520-539 60 60 60 104 104 104 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 60 60 60 102 102 102 T T T 29 580-599 59 59 59 101 101 101 F F F Max gen: 61, Max flush: 105 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 106 106 106 F F F 1 020-039 62 62 62 106 106 106 F F F 2 040-059 62 62 62 106 106 106 F F F 3 060-079 60 60 60 103 103 103 F F F 4 080-099 62 62 62 106 106 106 F F F 5 100-119 62 62 62 106 106 106 F F F 6 120-139 62 62 62 106 106 106 F F F 7 140-159 62 62 62 106 106 106 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 61 61 61 105 105 105 F F F 10 200-219 60 60 60 104 104 104 F F F 11 220-239 62 62 62 106 106 106 F F F 12 240-259 62 62 62 106 106 106 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 57 57 57 97 97 97 F F F 16 320-339 57 57 57 97 97 97 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 57 57 57 97 97 97 F F F 19 380-399 62 62 62 106 106 106 F F F 20 400-419 62 62 62 106 106 106 F F F 21 420-439 60 60 60 102 102 102 F F F 22 440-459 60 60 60 102 102 102 F F F 23 460-479 60 60 60 104 104 104 F F F 24 480-499 62 62 62 106 106 106 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 60 60 60 104 104 104 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 62 62 62 106 106 106 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 62, Max flush: 106 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.582241468Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:20.626400677Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.626813173Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.658974539Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.659582572Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.659616124Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.659629791Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.659643988Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440} {"msg":"Crucible 191b17ae-eb27-445d-a83e-a38102c44960 has session id: e05af391-e62f-4788-a127-050c58635ff0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660367244Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Upstairs opts: Upstairs UUID: 191b17ae-eb27-445d-a83e-a38102c44960, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660400516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Crucible stats registered with UUID: 191b17ae-eb27-445d-a83e-a38102c44960","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660415323Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:20.66042865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660843746Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660867531Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440} {"msg":"191b17ae-eb27-445d-a83e-a38102c44960 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660933066Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660950822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"0","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660974676Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"0","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.660998971Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"1","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661026654Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"1","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661053399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"2","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661082252Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"2","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661099058Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661117414Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661131051Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44631)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661144558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61641)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661169342Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"ds_connection connected from Ok(127.0.0.1:41491)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661194767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661866414Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"1","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661898087Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"2","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.661916413Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"0","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.66250351Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"1","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.662588071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"0","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.662664274Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"2","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663111972Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663222807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[0]R flush_numbers[0..12]: [106, 106, 106, 103, 106, 106, 106, 106, 106, 105, 104, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663285113Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[0]R generation[0..12]: [62, 62, 62, 60, 62, 62, 62, 62, 62, 61, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663304219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663323434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[1]R flush_numbers[0..12]: [106, 106, 106, 103, 106, 106, 106, 106, 106, 105, 104, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663337641Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[1]R generation[0..12]: [62, 62, 62, 60, 62, 62, 62, 62, 62, 61, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663351908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663365625Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[2]R flush_numbers[0..12]: [106, 106, 106, 103, 106, 106, 106, 106, 106, 105, 104, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663379142Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[2]R generation[0..12]: [62, 62, 62, 60, 62, 62, 62, 62, 62, 61, 60, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663392219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663405786Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Max found gen is 63","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663418313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Generation requested: 64 >= found:63","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.6634311Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663444067Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663456874Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"":"downstairs","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO ":30,"time":"2025-01-15T05:21:20.663538456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663557201Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"0","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663572028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"1","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663588005Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"client":"2","":"downstairs","session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663601131Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663613259Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"191b17ae-eb27-445d-a83e-a38102c44960 is now active with session: e05af391-e62f-4788-a127-050c58635ff0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663626656Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.663638253Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440,"session_id":"e05af391-e62f-4788-a127-050c58635ff0"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:20.66365023Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 32 len 24576 data: 6 5 4 4 3 4 02/30 Read block 204 len 20480 03/30 Read block 459 len 4096 04/30 Read block 366 len 28672 05/30 Read block 432 len 20480 06/30 Read block 221 len 24576 07/30 Flush 08/30 Write block 411 len 40960 data: 4 3 3 3 2 2 2 3 3 5 09/30 Write block 314 len 32768 data: 4 4 4 5 5 6 7 10 10/30 Read block 577 len 40960 11/30 Flush 12/30 Write block 568 len 12288 data: 5 6 5 13/30 Read block 372 len 28672 14/30 Write block 146 len 16384 data: 9 8 8 8 15/30 Read block 243 len 40960 16/30 Write block 315 len 36864 data: 5 5 6 6 7 8 11 9 9 17/30 Read block 380 len 36864 18/30 Read block 21 len 8192 19/30 Write block 63 len 32768 data: 4 4 5 4 5 7 7 8 20/30 Write block 399 len 20480 data: 4 4 5 7 7 21/30 Flush 22/30 Write block 85 len 20480 data: 6 5 6 7 8 23/30 Read block 509 len 32768 24/30 Write block 211 len 12288 data: 6 6 6 25/30 Flush 26/30 Write block 361 len 24576 data: 5 4 5 7 7 4 27/30 Read block 225 len 16384 28/30 Read block 406 len 28672 29/30 Write block 40 len 8192 data: 7 7 30/30 Write block 90 len 28672 data: 8 8 9 8 9 8 6 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:20.684452439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1440} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 106 106 106 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 62 62 62 106 106 106 T T T 3 060-079 64 64 64 109 109 109 F F F 4 080-099 64 64 64 110 110 110 T T T 5 100-119 62 62 62 106 106 106 F F F 6 120-139 62 62 62 106 106 106 F F F 7 140-159 64 64 64 109 109 109 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 61 61 61 105 105 105 F F F 10 200-219 64 64 64 110 110 110 F F F 11 220-239 62 62 62 106 106 106 F F F 12 240-259 62 62 62 106 106 106 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 64 64 64 109 109 109 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 57 57 57 97 97 97 T T T 19 380-399 64 64 64 109 109 109 F F F 20 400-419 64 64 64 109 109 109 F F F 21 420-439 64 64 64 108 108 108 F F F 22 440-459 60 60 60 102 102 102 F F F 23 460-479 60 60 60 104 104 104 F F F 24 480-499 62 62 62 106 106 106 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 60 60 60 104 104 104 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 64, Max flush: 110 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 106 106 106 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 65 65 65 111 111 111 F F F 3 060-079 64 64 64 109 109 109 F F F 4 080-099 65 65 65 111 111 111 F F F 5 100-119 62 62 62 106 106 106 F F F 6 120-139 62 62 62 106 106 106 F F F 7 140-159 64 64 64 109 109 109 F F F 8 160-179 62 62 62 106 106 106 F F F 9 180-199 61 61 61 105 105 105 F F F 10 200-219 64 64 64 110 110 110 F F F 11 220-239 62 62 62 106 106 106 F F F 12 240-259 62 62 62 106 106 106 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 64 64 64 109 109 109 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 64 64 64 109 109 109 F F F 20 400-419 64 64 64 109 109 109 F F F 21 420-439 64 64 64 108 108 108 F F F 22 440-459 60 60 60 102 102 102 F F F 23 460-479 60 60 60 104 104 104 F F F 24 480-499 62 62 62 106 106 106 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 60 60 60 104 104 104 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 65, Max flush: 111 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:24.954972297Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.001135042Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.001958775Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.002027939Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.002048285Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.002065961Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448} {"msg":"Crucible dc04a584-c1e9-4778-a813-85da1294f017 has session id: 0c7e6730-8324-484c-8be5-9fc982d7b6b6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.002679532Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Upstairs opts: Upstairs UUID: dc04a584-c1e9-4778-a813-85da1294f017, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.002728181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Crucible stats registered with UUID: dc04a584-c1e9-4778-a813-85da1294f017","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.002747367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:25.002763523Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003295712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003319957Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448} {"msg":"dc04a584-c1e9-4778-a813-85da1294f017 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003401299Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003419594Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"0","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003442879Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"0","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003459055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"1","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003488289Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"1","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003516282Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"2","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003544376Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"2","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00357004Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003598334Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00361568Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35164)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003678016Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:45647)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00370451Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35878)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.003728924Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.004513896Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"1","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00454121Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"0","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.004567994Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"2","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005054604Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"1","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005095745Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"0","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00520416Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"2","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00564665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005781659Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[0]R flush_numbers[0..12]: [106, 107, 111, 109, 111, 106, 106, 109, 106, 105, 110, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005859482Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[0]R generation[0..12]: [62, 64, 65, 64, 65, 62, 62, 64, 62, 61, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005881127Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005896643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[1]R flush_numbers[0..12]: [106, 107, 111, 109, 111, 106, 106, 109, 106, 105, 110, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00591247Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[1]R generation[0..12]: [62, 64, 65, 64, 65, 62, 62, 64, 62, 61, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005926367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005940483Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[2]R flush_numbers[0..12]: [106, 107, 111, 109, 111, 106, 106, 109, 106, 105, 110, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00595409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[2]R generation[0..12]: [62, 64, 65, 64, 65, 62, 62, 64, 62, 61, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005967597Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005981504Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.005994271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006024574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00604233Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"Wait for a query_work_queue command to finish before sending IO msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006062896Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"":"downstairs","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006076242Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.0060881Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"0","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006101507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"1","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006114354Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"client":"2","":"downstairs","session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006128461Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006140658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"dc04a584-c1e9-4778-a813-85da1294f017 is now active with session: 0c7e6730-8324-484c-8be5-9fc982d7b6b6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006153345Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.006164993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448,"session_id":"0c7e6730-8324-484c-8be5-9fc982d7b6b6"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:25.00617697Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 248 len 8192 data: 5 6 02/30 Read block 71 len 8192 03/30 Read block 239 len 32768 04/30 Read block 193 len 4096 05/30 Read block 61 len 4096 06/30 Read block 410 len 32768 07/30 Write block 190 len 12288 data: 9 9 10 08/30 Write block 416 len 4096 data: 3 09/30 Write block 175 len 40960 data: 7 7 7 7 8 9 9 8 7 6 10/30 Read block 19 len 20480 11/30 Flush 12/30 Write block 69 len 12288 data: 8 9 9 13/30 Read block 192 len 24576 14/30 Flush 15/30 Write block 415 len 36864 data: 3 4 3 4 4 6 4 4 5 16/30 Read block 182 len 4096 17/30 Read block 262 len 4096 18/30 Write block 107 len 8192 data: 7 8 19/30 Write block 2 len 20480 data: 5 7 7 6 6 20/30 Read block 589 len 8192 21/30 Read block 479 len 36864 22/30 Read block 585 len 4096 23/30 Read block 239 len 36864 24/30 Write block 527 len 36864 data: 7 7 8 7 5 3 3 4 4 25/30 Flush 26/30 Flush 27/30 Write block 54 len 32768 data: 3 3 4 4 4 4 4 5 28/30 Read block 188 len 4096 29/30 Read block 181 len 28672 30/30 Read block 101 len 20480 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:25.025442467Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1448} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 114 114 114 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 65 65 65 111 111 111 T T T 3 060-079 66 66 66 113 113 113 T T T 4 080-099 65 65 65 111 111 111 F F F 5 100-119 66 66 66 114 114 114 F F F 6 120-139 62 62 62 106 106 106 F F F 7 140-159 64 64 64 109 109 109 F F F 8 160-179 66 66 66 112 112 112 F F F 9 180-199 66 66 66 112 112 112 F F F 10 200-219 64 64 64 110 110 110 F F F 11 220-239 62 62 62 106 106 106 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 64 64 64 109 109 109 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 64 64 64 109 109 109 F F F 20 400-419 66 66 66 114 114 114 F F F 21 420-439 66 66 66 114 114 114 F F F 22 440-459 60 60 60 102 102 102 F F F 23 460-479 60 60 60 104 104 104 F F F 24 480-499 62 62 62 106 106 106 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 66 66 66 114 114 114 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 66, Max flush: 114 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 114 114 114 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 67 67 67 115 115 115 F F F 3 060-079 67 67 67 115 115 115 F F F 4 080-099 65 65 65 111 111 111 F F F 5 100-119 66 66 66 114 114 114 F F F 6 120-139 62 62 62 106 106 106 F F F 7 140-159 64 64 64 109 109 109 F F F 8 160-179 66 66 66 112 112 112 F F F 9 180-199 66 66 66 112 112 112 F F F 10 200-219 64 64 64 110 110 110 F F F 11 220-239 62 62 62 106 106 106 F F F 12 240-259 66 66 66 112 112 112 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 64 64 64 109 109 109 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 64 64 64 109 109 109 F F F 20 400-419 66 66 66 114 114 114 F F F 21 420-439 66 66 66 114 114 114 F F F 22 440-459 60 60 60 102 102 102 F F F 23 460-479 60 60 60 104 104 104 F F F 24 480-499 62 62 62 106 106 106 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 66 66 66 114 114 114 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 67, Max flush: 115 Begin loop 34 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.248459393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:29.292114796Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.292514976Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.324794495Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.325482499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.325517871Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.325532388Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.325546624Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456} {"msg":"Crucible e21ab5f8-526d-4b7f-b81a-0b5bebf87a2f has session id: 00032aa0-524f-47ae-a7fe-63e2b186a954","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326249275Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Upstairs opts: Upstairs UUID: e21ab5f8-526d-4b7f-b81a-0b5bebf87a2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326290456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Crucible stats registered with UUID: e21ab5f8-526d-4b7f-b81a-0b5bebf87a2f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326307842Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:29.326321159Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326678358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326701873Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456} {"msg":"e21ab5f8-526d-4b7f-b81a-0b5bebf87a2f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326764999Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326782724Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"0","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326805559Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"0","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326829164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"1","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326856358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"1","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326945858Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"2","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.32697907Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"2","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.326994267Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.327044625Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.327071929Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"ds_connection connected from Ok(127.0.0.1:55421)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.327166917Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"ds_connection connected from Ok(127.0.0.1:59051)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.327194361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"ds_connection connected from Ok(127.0.0.1:61584)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.327258077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.327665055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"2","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.327808292Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"1","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.327838716Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"0","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.32830491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"2","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.328334123Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"1","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.328409256Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"0","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.328883539Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329007021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[0]R flush_numbers[0..12]: [114, 107, 115, 115, 111, 114, 106, 109, 112, 112, 110, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329049761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[0]R generation[0..12]: [66, 64, 67, 67, 65, 66, 62, 64, 66, 66, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329067667Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329082184Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[1]R flush_numbers[0..12]: [114, 107, 115, 115, 111, 114, 106, 109, 112, 112, 110, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329096071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[1]R generation[0..12]: [66, 64, 67, 67, 65, 66, 62, 64, 66, 66, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329110787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329124844Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[2]R flush_numbers[0..12]: [114, 107, 115, 115, 111, 114, 106, 109, 112, 112, 110, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.32914297Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[2]R generation[0..12]: [66, 64, 67, 67, 65, 66, 62, 64, 66, 66, 64, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329157087Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329170804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329183311Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329196218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329213224Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329236059Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E"Wait for a query_work_queue command to finish before sending IO ,"pid":1456,"":"downstairs","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329263493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.32927639Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"0","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329289687Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"1","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329303824Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"client":"2","":"downstairs","session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329316701Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329328658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"e21ab5f8-526d-4b7f-b81a-0b5bebf87a2f is now active with session: 00032aa0-524f-47ae-a7fe-63e2b186a954","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329341275Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.329352813Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456,"session_id":"00032aa0-524f-47ae-a7fe-63e2b186a954"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:29.32936452Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 245 len 8192 data: 4 4 02/30 Read block 67 len 12288 03/30 Write block 136 len 16384 data: 5 4 4 2 04/30 Read block 299 len 12288 05/30 Read block 231 len 8192 06/30 Flush 07/30 Write block 128 len 28672 data: 4 4 5 4 4 3 3 08/30 Write block 454 len 36864 data: 6 5 4 4 4 5 3 3 4 09/30 Read block 253 len 32768 10/30 Read block 377 len 40960 11/30 Write block 437 len 24576 data: 6 7 6 5 6 5 12/30 Read block 521 len 36864 13/30 Read block 159 len 40960 14/30 Write block 486 len 16384 data: 8 10 10 6 15/30 Read block 74 len 20480 16/30 Read block 328 len 24576 17/30 Write block 118 len 8192 data: 7 5 18/30 Flush 19/30 Write block 439 len 24576 data: 7 6 7 6 6 6 20/30 Write block 241 len 24576 data: 10 10 8 6 5 5 21/30 Read block 62 len 12288 22/30 Write block 148 len 16384 data: 9 9 7 5 23/30 Write block 411 len 28672 data: 5 4 4 4 4 5 4 24/30 Write block 220 len 16384 data: 6 6 8 10 25/30 Write block 96 len 20480 data: 7 6 5 4 5 26/30 Write block 442 len 12288 data: 7 7 7 27/30 Write block 405 len 24576 data: 7 3 5 4 4 4 28/30 Write block 476 len 8192 data: 5 3 29/30 Write block 114 len 40960 data: 5 8 9 8 8 6 6 5 5 5 30/30 Write block 48 len 12288 data: 6 5 3 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:29.346654294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1456} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 114 114 114 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 67 67 67 115 115 115 T T T 3 060-079 67 67 67 115 115 115 F F F 4 080-099 65 65 65 111 111 111 T T T 5 100-119 68 68 68 117 117 117 T T T 6 120-139 68 68 68 117 117 117 T T T 7 140-159 64 64 64 109 109 109 T T T 8 160-179 66 66 66 112 112 112 F F F 9 180-199 66 66 66 112 112 112 F F F 10 200-219 64 64 64 110 110 110 F F F 11 220-239 62 62 62 106 106 106 T T T 12 240-259 68 68 68 116 116 116 T T T 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 64 64 64 109 109 109 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 64 64 64 109 109 109 F F F 20 400-419 66 66 66 114 114 114 T T T 21 420-439 68 68 68 117 117 117 T T T 22 440-459 68 68 68 117 117 117 T T T 23 460-479 68 68 68 117 117 117 T T T 24 480-499 68 68 68 117 117 117 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 66 66 66 114 114 114 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 68, Max flush: 117 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 114 114 114 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 69 69 69 118 118 118 F F F 3 060-079 67 67 67 115 115 115 F F F 4 080-099 69 69 69 118 118 118 F F F 5 100-119 69 69 69 118 118 118 F F F 6 120-139 69 69 69 118 118 118 F F F 7 140-159 69 69 69 118 118 118 F F F 8 160-179 66 66 66 112 112 112 F F F 9 180-199 66 66 66 112 112 112 F F F 10 200-219 64 64 64 110 110 110 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 69 69 69 118 118 118 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 64 64 64 109 109 109 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 65 65 65 111 111 111 F F F 19 380-399 64 64 64 109 109 109 F F F 20 400-419 69 69 69 118 118 118 F F F 21 420-439 69 69 69 118 118 118 F F F 22 440-459 69 69 69 118 118 118 F F F 23 460-479 69 69 69 118 118 118 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 62 62 62 106 106 106 F F F 26 520-539 66 66 66 114 114 114 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 59 59 59 101 101 101 F F F Max gen: 69, Max flush: 118 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.654558795Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:33.69908582Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.699513973Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.732403774Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.733109205Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.733150605Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.73317393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.733192006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464} {"msg":"Crucible 774e59ee-ef73-4c9b-8119-882a7ec54749 has session id: 92ba81e6-6337-4b34-8d1e-efdff581a1ef","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.73374545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Upstairs opts: Upstairs UUID: 774e59ee-ef73-4c9b-8119-882a7ec54749, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.733800348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Crucible stats registered with UUID: 774e59ee-ef73-4c9b-8119-882a7ec54749","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.733829851Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:33.733850017Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.73427714Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734300155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464} {"msg":"774e59ee-ef73-4c9b-8119-882a7ec54749 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734367769Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734385385Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"0","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.73440839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"0","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734432195Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"1","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734459349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"1","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734493211Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"2","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734521674Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"2","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734546789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734572953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734600427Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:36318)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734628201Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:53001)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734656344Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:37917)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.734827965Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.735316734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"2","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.735344978Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"1","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.735361694Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"0","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.735935624Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"1","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.735966497Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"2","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.735982864Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"0","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736474402Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736504696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[0]R flush_numbers[0..12]: [114, 107, 118, 115, 118, 118, 118, 118, 112, 112, 110, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736601803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[0]R generation[0..12]: [66, 64, 69, 67, 69, 69, 69, 69, 66, 66, 64, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736621659Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736636306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[1]R flush_numbers[0..12]: [114, 107, 118, 115, 118, 118, 118, 118, 112, 112, 110, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736650552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[1]R generation[0..12]: [66, 64, 69, 67, 69, 69, 69, 69, 66, 66, 64, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736664229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736678066Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[2]R flush_numbers[0..12]: [114, 107, 118, 115, 118, 118, 118, 118, 112, 112, 110, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736691913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[2]R generation[0..12]: [66, 64, 69, 67, 69, 69, 69, 69, 66, 66, 64, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736711849Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736737613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736763357Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.73679463Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736817425Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736831611Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"":"downstairs","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736853317Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736875701Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"0","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736901995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"1","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736922011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"client":"2","":"downstairs","session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736935638Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736954823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"774e59ee-ef73-4c9b-8119-882a7ec54749 is now active with session: 92ba81e6-6337-4b34-8d1e-efdff581a1ef","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.736977618Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.737000683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464,"session_id":"92ba81e6-6337-4b34-8d1e-efdff581a1ef"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:33.737023808Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 388 len 32768 data: 8 7 8 6 7 8 6 6 02/30 Write block 97 len 36864 data: 7 6 5 6 7 7 6 4 5 03/30 Read block 518 len 36864 04/30 Read block 577 len 16384 05/30 Read block 40 len 8192 06/30 Read block 119 len 20480 07/30 Write block 515 len 32768 data: 9 10 9 9 11 10 9 10 08/30 Flush 09/30 Write block 304 len 4096 data: 6 10/30 Read block 303 len 36864 11/30 Flush 12/30 Write block 245 len 16384 data: 6 6 4 6 13/30 Write block 584 len 16384 data: 8 7 6 5 14/30 Read block 395 len 16384 15/30 Read block 275 len 8192 16/30 Flush 17/30 Write block 177 len 28672 data: 8 8 9 10 10 9 8 18/30 Read block 580 len 28672 19/30 Read block 272 len 36864 20/30 Write block 377 len 8192 data: 7 7 21/30 Write block 211 len 36864 data: 7 7 7 5 4 6 7 8 7 22/30 Write block 411 len 8192 data: 6 5 23/30 Write block 173 len 28672 data: 8 8 8 8 9 9 10 24/30 Read block 165 len 28672 25/30 Flush 26/30 Flush 27/30 Write block 413 len 20480 data: 5 5 5 6 5 28/30 Write block 86 len 28672 data: 6 7 8 9 9 9 10 29/30 Read block 496 len 24576 30/30 Read block 517 len 36864 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:33.756402349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1464} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 114 114 114 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 69 69 69 118 118 118 F F F 3 060-079 67 67 67 115 115 115 F F F 4 080-099 70 70 70 119 119 119 T T T 5 100-119 70 70 70 119 119 119 F F F 6 120-139 69 69 69 118 118 118 F F F 7 140-159 69 69 69 118 118 118 F F F 8 160-179 70 70 70 122 122 122 F F F 9 180-199 70 70 70 122 122 122 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 70 70 70 121 121 121 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 70 70 70 119 119 119 F F F 20 400-419 70 70 70 122 122 122 T T T 21 420-439 69 69 69 118 118 118 F F F 22 440-459 69 69 69 118 118 118 F F F 23 460-479 69 69 69 118 118 118 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 70 70 70 119 119 119 F F F 26 520-539 70 70 70 119 119 119 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 70 70 70 121 121 121 F F F Max gen: 70, Max flush: 122 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 114 114 114 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 69 69 69 118 118 118 F F F 3 060-079 67 67 67 115 115 115 F F F 4 080-099 71 71 71 123 123 123 F F F 5 100-119 70 70 70 119 119 119 F F F 6 120-139 69 69 69 118 118 118 F F F 7 140-159 69 69 69 118 118 118 F F F 8 160-179 70 70 70 122 122 122 F F F 9 180-199 70 70 70 122 122 122 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 70 70 70 121 121 121 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 70 70 70 119 119 119 F F F 20 400-419 71 71 71 123 123 123 F F F 21 420-439 69 69 69 118 118 118 F F F 22 440-459 69 69 69 118 118 118 F F F 23 460-479 69 69 69 118 118 118 F F F 24 480-499 68 68 68 117 117 117 F F F 25 500-519 70 70 70 119 119 119 F F F 26 520-539 70 70 70 119 119 119 F F F 27 540-559 60 60 60 104 104 104 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 70 70 70 121 121 121 F F F Max gen: 71, Max flush: 123 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.016100206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.061894524Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.062566612Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.062610462Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.062625739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.062640185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472} {"msg":"Crucible a2dc65ef-e4fb-4501-bcc4-1e97a9b2e46e has session id: deaa2a6e-f5ee-4217-9d99-fec7054d81fd","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063339247Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Upstairs opts: Upstairs UUID: a2dc65ef-e4fb-4501-bcc4-1e97a9b2e46e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063383847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Crucible stats registered with UUID: a2dc65ef-e4fb-4501-bcc4-1e97a9b2e46e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063407501Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:38.063428527Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063800193Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063834525Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472} {"msg":"a2dc65ef-e4fb-4501-bcc4-1e97a9b2e46e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063883384Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063908868Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"0","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063946419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"0","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063973443Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"1","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.063999278Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"1","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064025262Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"2","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064045497Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"2","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064059424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064072931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064086298Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:40418)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064106953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:59083)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064156812Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58924)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064347579Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064942784Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"1","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.064989233Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"0","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.065230159Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"2","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.065696523Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"0","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.065736454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"1","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.065770866Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"2","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066279151Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066305555Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[0]R flush_numbers[0..12]: [114, 107, 118, 115, 123, 119, 118, 118, 122, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066381838Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[0]R generation[0..12]: [66, 64, 69, 67, 71, 70, 69, 69, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066402803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.06641767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[1]R flush_numbers[0..12]: [114, 107, 118, 115, 123, 119, 118, 118, 122, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066432756Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[1]R generation[0..12]: [66, 64, 69, 67, 71, 70, 69, 69, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066446823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.06646115Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[2]R flush_numbers[0..12]: [114, 107, 118, 115, 123, 119, 118, 118, 122, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066474817Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[2]R generation[0..12]: [66, 64, 69, 67, 71, 70, 69, 69, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066488284Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066502141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066519327Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066533773Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066547241Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:21:38.066560007Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"":"downstairs","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066577344Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066589401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"0","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066602268Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"1","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066615195Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"client":"2","":"downstairs","session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066628212Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066640369Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"a2dc65ef-e4fb-4501-bcc4-1e97a9b2e46e is now active with session: deaa2a6e-f5ee-4217-9d99-fec7054d81fd","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066652926Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066664664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472,"session_id":"deaa2a6e-f5ee-4217-9d99-fec7054d81fd"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:38.066676591Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 138 len 24576 data: 5 3 4 5 5 5 02/30 Write block 449 len 16384 data: 7 9 8 8 03/30 Read block 14 len 20480 04/30 Read block 173 len 24576 05/30 Read block 560 len 32768 06/30 Write block 92 len 28672 data: 11 9 10 9 8 8 7 07/30 Read block 294 len 32768 08/30 Read block 475 len 32768 09/30 Write block 74 len 16384 data: 6 3 2 4 10/30 Write block 468 len 4096 data: 8 11/30 Read block 122 len 16384 12/30 Write block 520 len 28672 data: 11 10 11 9 9 9 9 13/30 Read block 356 len 28672 14/30 Read block 574 len 36864 15/30 Flush 16/30 Write block 585 len 32768 data: 8 7 6 5 5 5 4 3 17/30 Write block 398 len 32768 data: 6 5 5 6 8 8 7 8 18/30 Read block 139 len 8192 19/30 Flush 20/30 Write block 542 len 24576 data: 6 6 7 7 6 6 21/30 Write block 489 len 8192 data: 7 6 22/30 Read block 191 len 20480 23/30 Write block 496 len 36864 data: 6 6 7 7 9 9 8 7 6 24/30 Read block 168 len 28672 25/30 Read block 68 len 36864 26/30 Write block 7 len 28672 data: 6 6 6 4 4 5 4 27/30 Read block 79 len 32768 28/30 Write block 108 len 16384 data: 9 6 4 5 29/30 Read block 309 len 12288 30/30 Write block 340 len 24576 data: 7 6 4 4 4 4 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 6 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:38.087235585Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1472} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 66 66 66 114 114 114 T T T 1 020-039 64 64 64 107 107 107 F F F 2 040-059 69 69 69 118 118 118 F F F 3 060-079 72 72 72 124 124 124 F F F 4 080-099 72 72 72 124 124 124 F F F 5 100-119 70 70 70 119 119 119 T T T 6 120-139 72 72 72 124 124 124 F F F 7 140-159 72 72 72 124 124 124 F F F 8 160-179 70 70 70 122 122 122 F F F 9 180-199 70 70 70 122 122 122 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 70 70 70 121 121 121 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 62 62 62 106 106 106 T T T 18 360-379 70 70 70 122 122 122 F F F 19 380-399 72 72 72 125 125 125 F F F 20 400-419 72 72 72 125 125 125 F F F 21 420-439 69 69 69 118 118 118 F F F 22 440-459 72 72 72 124 124 124 F F F 23 460-479 72 72 72 124 124 124 F F F 24 480-499 68 68 68 117 117 117 T T T 25 500-519 70 70 70 119 119 119 T T T 26 520-539 72 72 72 124 124 124 F F F 27 540-559 60 60 60 104 104 104 T T T 28 560-579 64 64 64 109 109 109 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 72, Max flush: 125 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 126 126 126 F F F 1 020-039 64 64 64 107 107 107 F F F 2 040-059 69 69 69 118 118 118 F F F 3 060-079 72 72 72 124 124 124 F F F 4 080-099 72 72 72 124 124 124 F F F 5 100-119 73 73 73 126 126 126 F F F 6 120-139 72 72 72 124 124 124 F F F 7 140-159 72 72 72 124 124 124 F F F 8 160-179 70 70 70 122 122 122 F F F 9 180-199 70 70 70 122 122 122 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 70 70 70 121 121 121 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 64 64 64 109 109 109 F F F 17 340-359 73 73 73 126 126 126 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 72 72 72 125 125 125 F F F 20 400-419 72 72 72 125 125 125 F F F 21 420-439 69 69 69 118 118 118 F F F 22 440-459 72 72 72 124 124 124 F F F 23 460-479 72 72 72 124 124 124 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 73 73 73 126 126 126 F F F 26 520-539 72 72 72 124 124 124 F F F 27 540-559 73 73 73 126 126 126 F F F 28 560-579 64 64 64 109 109 109 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 73, Max flush: 126 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.332979473Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:42.37727636Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.377679339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.41021458Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.410889377Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.41092396Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.410937366Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.410951463Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480} {"msg":"Crucible 7fac139a-8918-4742-9b10-0f7b8b840099 has session id: f042fe89-76e4-4ae2-b790-f3a410f8519e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.411615633Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Upstairs opts: Upstairs UUID: 7fac139a-8918-4742-9b10-0f7b8b840099, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.411695525Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Crucible stats registered with UUID: 7fac139a-8918-4742-9b10-0f7b8b840099","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.411735446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:42.411754141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412227724Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412262886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480} {"msg":"7fac139a-8918-4742-9b10-0f7b8b840099 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412321493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412350077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"0","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.41237656Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"0","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412392257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"1","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412406654Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"1","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412420141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"2","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412434247Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"2","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412447364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.41246822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412482916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49364)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412702297Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64480)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412755885Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"ds_connection connected from Ok(127.0.0.1:63310)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.412832217Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.413520912Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"1","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.413559023Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"2","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.413575179Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"0","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414256685Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"1","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414285778Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"0","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414301175Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"2","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.41485057Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414880453Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[0]R flush_numbers[0..12]: [126, 107, 118, 124, 124, 126, 124, 124, 122, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.41489591Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[0]R generation[0..12]: [73, 64, 69, 72, 72, 73, 72, 72, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414910517Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414928713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[1]R flush_numbers[0..12]: [126, 107, 118, 124, 124, 126, 124, 124, 122, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414942819Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[1]R generation[0..12]: [73, 64, 69, 72, 72, 73, 72, 72, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414956296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414969923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[2]R flush_numbers[0..12]: [126, 107, 118, 124, 124, 126, 124, 124, 122, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.41498341Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[2]R generation[0..12]: [73, 64, 69, 72, 72, 73, 72, 72, 70, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.414996617Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415010114Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415022891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415035748Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415048266Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415060733Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"":"downstairs","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.4150737Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415085457Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0EWait for a query_work_queue command to finish before sending IO ","pid":1480,"client":"0","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415107332Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"1","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415134826Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"client":"2","":"downstairs","session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415153002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415165239Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"7fac139a-8918-4742-9b10-0f7b8b840099 is now active with session: f042fe89-76e4-4ae2-b790-f3a410f8519e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415177916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415191123Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480,"session_id":"f042fe89-76e4-4ae2-b790-f3a410f8519e"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:42.415211889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 555 len 40960 data: 3 2 2 3 3 5 4 7 7 5 02/30 Write block 334 len 36864 data: 5 7 7 5 6 7 8 7 5 03/30 Write block 95 len 28672 data: 10 9 9 8 6 7 8 04/30 Read block 273 len 32768 05/30 Flush 06/30 Write block 405 len 8192 data: 9 4 07/30 Write block 61 len 40960 data: 6 5 5 5 6 5 6 8 9 10 08/30 Read block 315 len 20480 09/30 Read block 568 len 12288 10/30 Write block 63 len 36864 data: 6 6 7 6 7 9 10 11 10 11/30 Read block 402 len 20480 12/30 Read block 509 len 28672 13/30 Read block 511 len 36864 14/30 Write block 439 len 28672 data: 8 7 8 8 8 8 6 15/30 Write block 64 len 32768 data: 7 8 7 8 10 11 12 11 16/30 Write block 440 len 20480 data: 8 9 9 9 9 17/30 Write block 172 len 4096 data: 7 18/30 Read block 452 len 28672 19/30 Write block 504 len 20480 data: 7 4 5 4 5 20/30 Read block 196 len 24576 21/30 Read block 180 len 32768 22/30 Write block 26 len 12288 data: 7 6 7 23/30 Read block 232 len 40960 24/30 Read block 517 len 16384 25/30 Write block 47 len 32768 data: 7 7 6 4 4 4 4 4 26/30 Read block 437 len 24576 27/30 Write block 410 len 8192 data: 5 7 28/30 Write block 134 len 12288 data: 4 4 6 29/30 Write block 571 len 4096 data: 5 30/30 Read block 399 len 12288 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 2 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 25 24 24 73 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:42.434643617Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1480} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 126 126 126 F F F 1 020-039 64 64 64 107 107 107 T T T 2 040-059 69 69 69 118 118 118 T T T 3 060-079 72 72 72 124 124 124 T T T 4 080-099 74 74 74 127 127 127 F F F 5 100-119 74 74 74 127 127 127 F F F 6 120-139 72 72 72 124 124 124 T T T 7 140-159 72 72 72 124 124 124 F F F 8 160-179 70 70 70 122 122 122 T T T 9 180-199 70 70 70 122 122 122 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 70 70 70 121 121 121 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 127 127 127 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 72 72 72 125 125 125 F F F 20 400-419 72 72 72 125 125 125 T T T 21 420-439 69 69 69 118 118 118 T T T 22 440-459 72 72 72 124 124 124 T T T 23 460-479 72 72 72 124 124 124 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 73 73 73 126 126 126 T T T 26 520-539 72 72 72 124 124 124 F F F 27 540-559 74 74 74 127 127 127 F F F 28 560-579 74 74 74 127 127 127 T T T 29 580-599 72 72 72 125 125 125 F F F Max gen: 74, Max flush: 127 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 126 126 126 F F F 1 020-039 75 75 75 128 128 128 F F F 2 040-059 75 75 75 128 128 128 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 74 74 74 127 127 127 F F F 5 100-119 74 74 74 127 127 127 F F F 6 120-139 75 75 75 128 128 128 F F F 7 140-159 72 72 72 124 124 124 F F F 8 160-179 75 75 75 128 128 128 F F F 9 180-199 70 70 70 122 122 122 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 70 70 70 121 121 121 F F F 13 260-279 59 59 59 101 101 101 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 70 70 70 120 120 120 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 127 127 127 F F F 18 360-379 70 70 70 122 122 122 F F F 19 380-399 72 72 72 125 125 125 F F F 20 400-419 75 75 75 128 128 128 F F F 21 420-439 75 75 75 128 128 128 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 72 72 72 124 124 124 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 75 75 75 128 128 128 F F F 26 520-539 72 72 72 124 124 124 F F F 27 540-559 74 74 74 127 127 127 F F F 28 560-579 75 75 75 128 128 128 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 75, Max flush: 128 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.73922946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:46.785622192Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.786025041Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.818872922Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.819708322Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.819744044Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.819757801Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.819771798Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488} {"msg":"Crucible 720f7987-d4a2-48c3-b9f2-2bfd52c93f88 has session id: b1c7e505-230f-4730-8b4e-436700a4876a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.820492075Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Upstairs opts: Upstairs UUID: 720f7987-d4a2-48c3-b9f2-2bfd52c93f88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.820550651Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Crucible stats registered with UUID: 720f7987-d4a2-48c3-b9f2-2bfd52c93f88","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.820572857Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:46.820590393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821056757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821082491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488} {"msg":"720f7987-d4a2-48c3-b9f2-2bfd52c93f88 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821147356Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821164922Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"0","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821188347Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"0","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821202214Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"1","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821216131Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"1","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821229338Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"2","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821243105Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"2","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821256122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821270579Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821293043Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"ds_connection connected from Ok(127.0.0.1:50673)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821430152Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65497)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821467414Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34373)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.821653272Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.822209026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"2","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.822248757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"1","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.82227742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"0","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.822868697Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"1","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.822910187Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"2","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.82294229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"0","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823474459Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823500803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[0]R flush_numbers[0..12]: [126, 128, 128, 128, 127, 127, 128, 124, 128, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823564709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[0]R generation[0..12]: [73, 75, 75, 75, 74, 74, 75, 72, 75, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823585014Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823599641Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[1]R flush_numbers[0..12]: [126, 128, 128, 128, 127, 127, 128, 124, 128, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823613428Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[1]R generation[0..12]: [73, 75, 75, 75, 74, 74, 75, 72, 75, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823626805Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823640822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[2]R flush_numbers[0..12]: [126, 128, 128, 128, 127, 127, 128, 124, 128, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823658028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[2]R generation[0..12]: [73, 75, 75, 75, 74, 74, 75, 72, 75, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823671665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823685381Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823697989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823710776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823723673Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.8237363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"":"downstairs","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823749167Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823760895Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"0","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823783839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"1","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823796606Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"client":"2","":"downstairs","session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823809583Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823821421Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"720f7987-d4a2-48c3-b9f2-2bfd52c93f88 is now active with session: b1c7e505-230f-4730-8b4e-436700a4876a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823833908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823845495Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488,"session_id":"b1c7e505-230f-4730-8b4e-436700a4876a"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:46.823859462Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 519 len 20480 data: 12 12 11 12 10 03/30 Read block 245 len 20480 04/30 Write block 401 len 32768 data: 7 9 9 8 10 5 6 5 05/30 Read block 560 len 12288 06/30 Write block 377 len 16384 data: 8 8 10 8 07/30 Read block 455 len 40960 08/30 Write block 256 len 20480 data: 4 3 4 5 5 09/30 Read block 0 len 16384 10/30 Read block 123 len 12288 11/30 Read block 561 len 36864 12/30 Write block 530 len 28672 data: 8 6 4 4 5 5 6 13/30 Read block 23 len 4096 14/30 Write block 81 len 36864 data: 9 10 10 10 7 7 8 9 10 15/30 Read block 394 len 12288 16/30 Read block 239 len 28672 17/30 Write block 165 len 24576 data: 7 7 6 5 6 5 18/30 Write block 410 len 24576 data: 6 8 6 6 6 6 19/30 Write block 368 len 16384 data: 4 3 4 4 20/30 Read block 207 len 16384 21/30 Write block 303 len 4096 data: 5 22/30 Read block 248 len 20480 23/30 Read block 574 len 28672 24/30 Read block 254 len 40960 25/30 Flush 26/30 Write block 157 len 28672 data: 11 9 10 10 9 10 10 27/30 Read block 588 len 16384 28/30 Read block 476 len 40960 29/30 Write block 568 len 12288 data: 6 7 6 30/30 Read block 29 len 24576 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Sent Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 3 5 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:46.844737424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1488} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 126 126 126 F F F 1 020-039 75 75 75 128 128 128 F F F 2 040-059 75 75 75 128 128 128 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 74 74 74 127 127 127 F F F 6 120-139 75 75 75 128 128 128 F F F 7 140-159 72 72 72 124 124 124 T T T 8 160-179 76 76 76 130 130 130 T T T 9 180-199 70 70 70 122 122 122 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 76 76 76 130 130 130 F F F 13 260-279 76 76 76 130 130 130 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 76 76 76 130 130 130 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 127 127 127 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 76 76 76 130 130 130 F F F 20 400-419 76 76 76 130 130 130 F F F 21 420-439 75 75 75 128 128 128 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 72 72 72 124 124 124 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 76 76 76 130 130 130 F F F 26 520-539 76 76 76 130 130 130 F F F 27 540-559 74 74 74 127 127 127 F F F 28 560-579 75 75 75 128 128 128 T T T 29 580-599 72 72 72 125 125 125 F F F Max gen: 76, Max flush: 130 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 126 126 126 F F F 1 020-039 75 75 75 128 128 128 F F F 2 040-059 75 75 75 128 128 128 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 76 76 76 130 130 130 F F F 5 100-119 74 74 74 127 127 127 F F F 6 120-139 75 75 75 128 128 128 F F F 7 140-159 77 77 77 131 131 131 F F F 8 160-179 77 77 77 131 131 131 F F F 9 180-199 70 70 70 122 122 122 F F F 10 200-219 70 70 70 122 122 122 F F F 11 220-239 69 69 69 118 118 118 F F F 12 240-259 76 76 76 130 130 130 F F F 13 260-279 76 76 76 130 130 130 F F F 14 280-299 52 52 52 88 88 88 F F F 15 300-319 76 76 76 130 130 130 F F F 16 320-339 74 74 74 127 127 127 F F F 17 340-359 74 74 74 127 127 127 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 76 76 76 130 130 130 F F F 20 400-419 76 76 76 130 130 130 F F F 21 420-439 75 75 75 128 128 128 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 72 72 72 124 124 124 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 76 76 76 130 130 130 F F F 26 520-539 76 76 76 130 130 130 F F F 27 540-559 74 74 74 127 127 127 F F F 28 560-579 77 77 77 131 131 131 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 77, Max flush: 131 Begin loop 39 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.108640239Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.153425447Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.154068001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.154110561Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.154126028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.154141105Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496} {"msg":"Crucible 8d51b0f9-a168-4958-8728-8ddde8dd8bfa has session id: 5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.154773001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Upstairs opts: Upstairs UUID: 8d51b0f9-a168-4958-8728-8ddde8dd8bfa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.154819381Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Crucible stats registered with UUID: 8d51b0f9-a168-4958-8728-8ddde8dd8bfa","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.154844455Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:51.15486568Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155329735Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155360618Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496} {"msg":"8d51b0f9-a168-4958-8728-8ddde8dd8bfa active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155451348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155470563Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"0","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155494808Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"0","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155508875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"1","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155523441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"1","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155546716Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"2","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155569821Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"2","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155593545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155625118Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155652982Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60271)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155678386Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57843)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155703291Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:63309)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.155880651Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.156330829Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"1","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.156486673Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"0","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.156527894Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"2","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.156972633Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"1","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.157095395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"0","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.157349238Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"2","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.157671595Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.157696809Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[0]R flush_numbers[0..12]: [126, 128, 128, 128, 130, 127, 128, 131, 131, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.157911751Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[0]R generation[0..12]: [73, 75, 75, 75, 76, 74, 75, 77, 77, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.157948112Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.157976486Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[1]R flush_numbers[0..12]: [126, 128, 128, 128, 130, 127, 128, 131, 131, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158000131Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[1]R generation[0..12]: [73, 75, 75, 75, 76, 74, 75, 77, 77, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158024185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158049439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[2]R flush_numbers[0..12]: [126, 128, 128, 128, 130, 127, 128, 131, 131, 122, 122, 118]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158075473Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[2]R generation[0..12]: [73, 75, 75, 75, 76, 74, 75, 77, 77, 70, 70, 69]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158100208Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158125912Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158152006Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158185149Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158212083Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158238597Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"":"downstairs","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158264951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158288985Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"0","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158314779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"1","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158345052Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"client":"2","":"downstairs","session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158376245Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.15840013Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"8d51b0f9-a168-4958-8728-8ddde8dd8bfa is now active with session: 5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158422755Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158435912Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496,"session_id":"5dfc5a7b-9c81-441a-bf3a-f57117cbcfb6"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:51.158448039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 224 len 28672 data: 8 9 9 9 9 8 8 03/30 Write block 541 len 28672 data: 4 7 7 8 8 7 7 04/30 Write block 202 len 4096 data: 9 05/30 Read block 279 len 24576 06/30 Write block 522 len 4096 data: 13 07/30 Write block 5 len 24576 data: 7 7 7 7 7 5 08/30 Write block 327 len 36864 data: 5 5 5 5 4 4 5 6 8 09/30 Read block 172 len 36864 10/30 Read block 230 len 40960 11/30 Write block 33 len 24576 data: 6 5 5 4 5 7 12/30 Write block 132 len 36864 data: 5 4 5 5 7 5 6 4 5 13/30 Write block 246 len 32768 data: 7 5 7 7 5 6 4 5 14/30 Write block 523 len 28672 data: 11 10 10 10 8 8 9 15/30 Flush 16/30 Write block 272 len 40960 data: 5 6 5 3 2 2 5 4 4 4 17/30 Write block 1 len 32768 data: 6 6 8 8 8 8 8 8 18/30 Read block 470 len 4096 19/30 Write block 261 len 32768 data: 5 3 5 7 7 8 8 9 20/30 Read block 347 len 24576 21/30 Read block 76 len 32768 22/30 Flush 23/30 Write block 435 len 16384 data: 7 6 7 8 24/30 Write block 130 len 16384 data: 6 5 6 5 25/30 Flush 26/30 Write block 191 len 4096 data: 10 27/30 Read block 178 len 12288 28/30 Write block 308 len 32768 data: 4 4 3 3 3 3 5 6 29/30 Write block 167 len 32768 data: 7 6 7 6 6 8 9 9 30/30 Write block 90 len 12288 data: 10 10 12 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:51.176830006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1496} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 134 134 134 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 75 75 75 128 128 128 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 76 76 76 130 130 130 T T T 5 100-119 74 74 74 127 127 127 F F F 6 120-139 78 78 78 135 135 135 F F F 7 140-159 78 78 78 133 133 133 F F F 8 160-179 77 77 77 131 131 131 T T T 9 180-199 70 70 70 122 122 122 T T T 10 200-219 78 78 78 133 133 133 F F F 11 220-239 78 78 78 133 133 133 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 78 78 78 134 134 134 F F F 14 280-299 78 78 78 134 134 134 F F F 15 300-319 76 76 76 130 130 130 T T T 16 320-339 78 78 78 133 133 133 F F F 17 340-359 74 74 74 127 127 127 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 76 76 76 130 130 130 F F F 20 400-419 76 76 76 130 130 130 F F F 21 420-439 78 78 78 135 135 135 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 72 72 72 124 124 124 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 76 76 76 130 130 130 F F F 26 520-539 78 78 78 133 133 133 F F F 27 540-559 78 78 78 133 133 133 F F F 28 560-579 77 77 77 131 131 131 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 78, Max flush: 135 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 134 134 134 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 75 75 75 128 128 128 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 79 79 79 136 136 136 F F F 5 100-119 74 74 74 127 127 127 F F F 6 120-139 78 78 78 135 135 135 F F F 7 140-159 78 78 78 133 133 133 F F F 8 160-179 79 79 79 136 136 136 F F F 9 180-199 79 79 79 136 136 136 F F F 10 200-219 78 78 78 133 133 133 F F F 11 220-239 78 78 78 133 133 133 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 78 78 78 134 134 134 F F F 14 280-299 78 78 78 134 134 134 F F F 15 300-319 79 79 79 136 136 136 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 74 74 74 127 127 127 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 76 76 76 130 130 130 F F F 20 400-419 76 76 76 130 130 130 F F F 21 420-439 78 78 78 135 135 135 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 72 72 72 124 124 124 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 76 76 76 130 130 130 F F F 26 520-539 78 78 78 133 133 133 F F F 27 540-559 78 78 78 133 133 133 F F F 28 560-579 77 77 77 131 131 131 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 79, Max flush: 136 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.412621348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:55.456084125Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.456444433Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.488484457Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.489074463Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.489109725Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.489123772Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.489137829Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504} {"msg":"Crucible 6202b992-3b92-40b3-ba09-760adb69937c has session id: 22d1ce5c-9315-43ab-8586-f8cfa1735337","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.489770296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Upstairs opts: Upstairs UUID: 6202b992-3b92-40b3-ba09-760adb69937c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.489853867Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Crucible stats registered with UUID: 6202b992-3b92-40b3-ba09-760adb69937c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.489891098Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:55.489920642Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490410491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490436265Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504} {"msg":"6202b992-3b92-40b3-ba09-760adb69937c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490508339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490526055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"0","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490553918Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"0","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490573564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"1","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490599148Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"1","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490623462Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"2","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490650426Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"2","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490674501Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490699155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490755143Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"ds_connection connected from Ok(127.0.0.1:39089)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490789875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"ds_connection connected from Ok(127.0.0.1:54884)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490819298Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"ds_connection connected from Ok(127.0.0.1:54385)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.490847002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.491521039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"1","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.491549973Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"2","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.491565569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"0","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492150527Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"2","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.49218034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"1","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492196206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"0","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492690834Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492803969Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[0]R flush_numbers[0..12]: [134, 133, 128, 128, 136, 127, 135, 133, 136, 136, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492830682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[0]R generation[0..12]: [78, 78, 75, 75, 79, 74, 78, 78, 79, 79, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492846129Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492860736Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[1]R flush_numbers[0..12]: [134, 133, 128, 128, 136, 127, 135, 133, 136, 136, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492874803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[1]R generation[0..12]: [78, 78, 75, 75, 79, 74, 78, 78, 79, 79, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492888739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492902496Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[2]R flush_numbers[0..12]: [134, 133, 128, 128, 136, 127, 135, 133, 136, 136, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492915933Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[2]R generation[0..12]: [78, 78, 75, 75, 79, 74, 78, 78, 79, 79, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.49292913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492942867Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492959463Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.49297295Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.492985767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:21:55.493013801Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"":"downstairs","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493034276Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493046294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"0","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493059491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"1","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493072218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"client":"2","":"downstairs","session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493085155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493097442Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"6202b992-3b92-40b3-ba09-760adb69937c is now active with session: 22d1ce5c-9315-43ab-8586-f8cfa1735337","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493110139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493121927Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504,"session_id":"22d1ce5c-9315-43ab-8586-f8cfa1735337"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:55.493133794Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 534 len 12288 data: 6 6 7 02/30 Write block 505 len 32768 data: 5 6 5 6 4 4 4 6 03/30 Flush 04/30 Write block 438 len 4096 data: 9 05/30 Write block 229 len 24576 data: 9 9 8 5 6 7 06/30 Read block 80 len 8192 07/30 Read block 384 len 12288 08/30 Read block 283 len 8192 09/30 Read block 245 len 12288 10/30 Write block 519 len 8192 data: 13 13 11/30 Read block 319 len 24576 12/30 Read block 130 len 36864 13/30 Flush 14/30 Write block 205 len 32768 data: 6 5 6 6 8 6 8 8 15/30 Write block 306 len 20480 data: 3 4 5 5 4 16/30 Read block 90 len 28672 17/30 Write block 261 len 4096 data: 6 18/30 Flush 19/30 Write block 348 len 36864 data: 8 9 7 7 8 10 10 10 11 20/30 Write block 169 len 4096 data: 8 21/30 Read block 127 len 16384 22/30 Read block 71 len 16384 23/30 Read block 10 len 24576 24/30 Write block 206 len 12288 data: 6 7 7 25/30 Write block 466 len 12288 data: 8 7 9 26/30 Write block 180 len 32768 data: 11 11 10 9 7 6 7 7 27/30 Read block 28 len 12288 28/30 Read block 271 len 32768 29/30 Write block 169 len 36864 data: 9 7 7 9 10 10 9 9 10 30/30 Read block 349 len 20480 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:55.511626076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1504} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 134 134 134 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 75 75 75 128 128 128 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 79 79 79 136 136 136 F F F 5 100-119 74 74 74 127 127 127 F F F 6 120-139 78 78 78 135 135 135 F F F 7 140-159 78 78 78 133 133 133 F F F 8 160-179 79 79 79 136 136 136 T T T 9 180-199 79 79 79 136 136 136 T T T 10 200-219 80 80 80 139 139 139 T T T 11 220-239 80 80 80 138 138 138 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 80 80 80 139 139 139 F F F 14 280-299 78 78 78 134 134 134 F F F 15 300-319 80 80 80 139 139 139 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 74 74 74 127 127 127 T T T 18 360-379 76 76 76 130 130 130 F F F 19 380-399 76 76 76 130 130 130 F F F 20 400-419 76 76 76 130 130 130 F F F 21 420-439 80 80 80 138 138 138 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 72 72 72 124 124 124 T T T 24 480-499 73 73 73 126 126 126 F F F 25 500-519 80 80 80 138 138 138 F F F 26 520-539 80 80 80 138 138 138 F F F 27 540-559 78 78 78 133 133 133 F F F 28 560-579 77 77 77 131 131 131 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 80, Max flush: 139 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 134 134 134 F F F 1 020-039 78 78 78 133 133 133 F F F 2 040-059 75 75 75 128 128 128 F F F 3 060-079 75 75 75 128 128 128 F F F 4 080-099 79 79 79 136 136 136 F F F 5 100-119 74 74 74 127 127 127 F F F 6 120-139 78 78 78 135 135 135 F F F 7 140-159 78 78 78 133 133 133 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 81 81 81 140 140 140 F F F 10 200-219 81 81 81 140 140 140 F F F 11 220-239 80 80 80 138 138 138 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 80 80 80 139 139 139 F F F 14 280-299 78 78 78 134 134 134 F F F 15 300-319 80 80 80 139 139 139 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 81 81 81 140 140 140 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 76 76 76 130 130 130 F F F 20 400-419 76 76 76 130 130 130 F F F 21 420-439 80 80 80 138 138 138 F F F 22 440-459 75 75 75 128 128 128 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 80 80 80 138 138 138 F F F 26 520-539 80 80 80 138 138 138 F F F 27 540-559 78 78 78 133 133 133 F F F 28 560-579 77 77 77 131 131 131 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 81, Max flush: 140 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.783329804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:59.827375448Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.827838034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.860355959Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.86101294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.86105824Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.861075016Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.861092822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512} {"msg":"Crucible 2de88c5b-9c64-4cb8-8b17-7dc3889e79f6 has session id: 3d52b715-8a61-4719-a1d9-fb333534ba4e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.861796363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Upstairs opts: Upstairs UUID: 2de88c5b-9c64-4cb8-8b17-7dc3889e79f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.861833284Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Crucible stats registered with UUID: 2de88c5b-9c64-4cb8-8b17-7dc3889e79f6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.861849971Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:59.861863368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862307367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862340869Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512} {"msg":"2de88c5b-9c64-4cb8-8b17-7dc3889e79f6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862395487Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862422101Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"0","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862445505Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"0","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862459912Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"1","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862475279Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"1","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862488886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"2","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862503083Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"2","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862516169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862529817Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862549582Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49210)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862657777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41812)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.86269286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58952)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.862898663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.863478891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"2","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.863516243Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"1","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.863584478Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"0","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864163916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"1","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864204717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"2","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.86423575Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"0","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864693956Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864817698Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[0]R flush_numbers[0..12]: [134, 133, 128, 128, 136, 127, 135, 133, 140, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864845892Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[0]R generation[0..12]: [78, 78, 75, 75, 79, 74, 78, 78, 81, 81, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864860998Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864879045Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[1]R flush_numbers[0..12]: [134, 133, 128, 128, 136, 127, 135, 133, 140, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864893221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[1]R generation[0..12]: [78, 78, 75, 75, 79, 74, 78, 78, 81, 81, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864906808Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864920565Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[2]R flush_numbers[0..12]: [134, 133, 128, 128, 136, 127, 135, 133, 140, 140, 140, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864934262Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[2]R generation[0..12]: [78, 78, 75, 75, 79, 74, 78, 78, 81, 81, 81, 80]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864947449Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864960996Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.864973453Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.86498634Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865002186Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865024821Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"":"downstairs","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865040738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:21:59.865052675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"0","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865071951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"1","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865084688Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"client":"2","":"downstairs","session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865100804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865112992Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"2de88c5b-9c64-4cb8-8b17-7dc3889e79f6 is now active with session: 3d52b715-8a61-4719-a1d9-fb333534ba4e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865125549Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865137336Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512,"session_id":"3d52b715-8a61-4719-a1d9-fb333534ba4e"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:21:59.865149223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 557 len 8192 data: 3 4 02/30 Write block 116 len 20480 data: 10 9 9 7 7 03/30 Read block 207 len 20480 04/30 Write block 78 len 24576 data: 4 5 7 10 11 11 05/30 Read block 452 len 12288 06/30 Read block 435 len 12288 07/30 Write block 102 len 36864 data: 8 7 5 6 6 8 10 7 5 08/30 Read block 375 len 32768 09/30 Write block 398 len 36864 data: 7 6 6 8 10 10 9 11 6 10/30 Read block 111 len 36864 11/30 Read block 196 len 36864 12/30 Write block 274 len 36864 data: 6 4 3 3 6 5 5 5 4 13/30 Read block 486 len 36864 14/30 Flush 15/30 Write block 36 len 20480 data: 5 6 8 7 8 16/30 Read block 361 len 8192 17/30 Read block 408 len 12288 18/30 Read block 173 len 32768 19/30 Write block 541 len 40960 data: 5 8 8 9 9 8 8 6 6 6 20/30 Write block 142 len 24576 data: 6 6 6 8 10 9 21/30 Read block 191 len 8192 22/30 Write block 572 len 24576 data: 5 4 4 4 6 6 23/30 Read block 519 len 36864 24/30 Write block 405 len 20480 data: 12 7 7 6 5 25/30 Read block 478 len 32768 26/30 Flush 27/30 Write block 142 len 20480 data: 7 7 7 9 11 28/30 Write block 442 len 20480 data: 10 10 10 7 4 29/30 Write block 222 len 40960 data: 9 11 9 10 10 10 10 10 10 9 30/30 Read block 366 len 20480 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:21:59.886756819Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1512} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 134 134 134 F F F 1 020-039 82 82 82 142 142 142 F F F 2 040-059 82 82 82 142 142 142 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 82 82 82 141 141 141 F F F 6 120-139 82 82 82 141 141 141 F F F 7 140-159 82 82 82 142 142 142 T T T 8 160-179 81 81 81 140 140 140 F F F 9 180-199 81 81 81 140 140 140 F F F 10 200-219 81 81 81 140 140 140 F F F 11 220-239 80 80 80 138 138 138 T T T 12 240-259 78 78 78 133 133 133 F F F 13 260-279 82 82 82 141 141 141 F F F 14 280-299 82 82 82 141 141 141 F F F 15 300-319 80 80 80 139 139 139 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 81 81 81 140 140 140 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 82 82 82 141 141 141 F F F 20 400-419 82 82 82 142 142 142 F F F 21 420-439 80 80 80 138 138 138 F F F 22 440-459 75 75 75 128 128 128 T T T 23 460-479 81 81 81 140 140 140 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 80 80 80 138 138 138 F F F 26 520-539 80 80 80 138 138 138 F F F 27 540-559 82 82 82 142 142 142 F F F 28 560-579 82 82 82 142 142 142 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 82, Max flush: 142 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 134 134 134 F F F 1 020-039 82 82 82 142 142 142 F F F 2 040-059 82 82 82 142 142 142 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 82 82 82 141 141 141 F F F 6 120-139 82 82 82 141 141 141 F F F 7 140-159 83 83 83 143 143 143 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 81 81 81 140 140 140 F F F 10 200-219 81 81 81 140 140 140 F F F 11 220-239 83 83 83 143 143 143 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 82 82 82 141 141 141 F F F 14 280-299 82 82 82 141 141 141 F F F 15 300-319 80 80 80 139 139 139 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 81 81 81 140 140 140 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 82 82 82 141 141 141 F F F 20 400-419 82 82 82 142 142 142 F F F 21 420-439 80 80 80 138 138 138 F F F 22 440-459 83 83 83 143 143 143 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 80 80 80 138 138 138 F F F 26 520-539 80 80 80 138 138 138 F F F 27 540-559 82 82 82 142 142 142 F F F 28 560-579 82 82 82 142 142 142 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 83, Max flush: 143 Begin loop 42 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.151486408Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:04.196172318Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.196569468Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.228987366Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.229578762Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.229614454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.22962856Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.229643367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520} {"msg":"Crucible a97e3374-f1b2-48ad-a1bb-d41918e2d0c2 has session id: 28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.230251569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Upstairs opts: Upstairs UUID: a97e3374-f1b2-48ad-a1bb-d41918e2d0c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.230303847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Crucible stats registered with UUID: a97e3374-f1b2-48ad-a1bb-d41918e2d0c2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.230328372Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:04.230348397Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.230875338Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.230902732Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520} {"msg":"a97e3374-f1b2-48ad-a1bb-d41918e2d0c2 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.230965758Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.230985183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"0","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231008668Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"0","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231027044Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"1","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.23104198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"1","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231055287Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"2","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231069294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"2","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231082401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231095708Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231109065Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45814)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231412806Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37971)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231450388Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:59476)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231466584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.23192861Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"2","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.231957243Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"1","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.23219012Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"0","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.232703304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"2","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.232747194Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"1","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.232840183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"0","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233299649Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233333232Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[0]R flush_numbers[0..12]: [134, 142, 142, 141, 141, 141, 141, 143, 140, 140, 140, 143]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233432339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[0]R generation[0..12]: [78, 82, 82, 82, 82, 82, 82, 83, 81, 81, 81, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233453424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233467861Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[1]R flush_numbers[0..12]: [134, 142, 142, 141, 141, 141, 141, 143, 140, 140, 140, 143]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233481738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[1]R generation[0..12]: [78, 82, 82, 82, 82, 82, 82, 83, 81, 81, 81, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233495605Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233509592Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[2]R flush_numbers[0..12]: [134, 142, 142, 141, 141, 141, 141, 143, 140, 140, 140, 143]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233523069Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[2]R generation[0..12]: [78, 82, 82, 82, 82, 82, 82, 83, 81, 81, 81, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233536246Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233549812Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233562709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233580165Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233600491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233613438Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"":"downstairs","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233626325Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233638152Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"0","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.23365112Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"1","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233664006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"client":"2","":"downstairs","session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233677054Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233689131Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"a97e3374-f1b2-48ad-a1bb-d41918e2d0c2 is now active with session: 28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233701818Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233713535Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520,"session_id":"28e03e5a-5eaf-47c3-8f85-27bddc6e6b5b"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:04.233725552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 509 len 32768 data: 5 5 5 7 9 9 10 11 02/30 Read block 539 len 36864 03/30 Read block 220 len 20480 04/30 Write block 220 len 40960 data: 7 7 10 12 10 11 11 11 11 11 05/30 Read block 105 len 8192 06/30 Write block 316 len 16384 data: 6 7 7 8 07/30 Read block 544 len 40960 08/30 Write block 575 len 8192 data: 5 7 09/30 Read block 514 len 36864 10/30 Read block 504 len 24576 11/30 Read block 348 len 16384 12/30 Write block 423 len 28672 data: 6 6 7 7 7 7 8 13/30 Flush 14/30 Write block 419 len 12288 data: 5 7 5 15/30 Read block 145 len 24576 16/30 Read block 29 len 4096 17/30 Write block 16 len 24576 data: 5 5 5 5 7 7 18/30 Read block 20 len 40960 19/30 Read block 427 len 16384 20/30 Read block 575 len 12288 21/30 Read block 228 len 12288 22/30 Read block 591 len 4096 23/30 Read block 201 len 24576 24/30 Read block 543 len 12288 25/30 Flush 26/30 Write block 385 len 32768 data: 9 8 8 9 8 9 7 8 27/30 Write block 194 len 24576 data: 8 7 7 6 6 7 28/30 Write block 569 len 24576 data: 8 7 6 6 5 5 29/30 Write block 404 len 20480 data: 10 13 8 8 7 30/30 Write block 108 len 40960 data: 11 8 6 6 5 6 6 9 11 10 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:04.254080613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1520} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 145 145 145 F F F 1 020-039 84 84 84 145 145 145 F F F 2 040-059 82 82 82 142 142 142 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 82 82 82 141 141 141 T T T 6 120-139 82 82 82 141 141 141 F F F 7 140-159 83 83 83 143 143 143 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 81 81 81 140 140 140 T T T 10 200-219 81 81 81 140 140 140 F F F 11 220-239 84 84 84 144 144 144 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 82 82 82 141 141 141 F F F 14 280-299 82 82 82 141 141 141 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 81 81 81 140 140 140 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 82 82 82 141 141 141 T T T 20 400-419 84 84 84 145 145 145 T T T 21 420-439 84 84 84 145 145 145 F F F 22 440-459 83 83 83 143 143 143 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 84 84 84 144 144 144 F F F 26 520-539 80 80 80 138 138 138 F F F 27 540-559 82 82 82 142 142 142 F F F 28 560-579 84 84 84 144 144 144 T T T 29 580-599 72 72 72 125 125 125 F F F Max gen: 84, Max flush: 145 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 145 145 145 F F F 1 020-039 84 84 84 145 145 145 F F F 2 040-059 82 82 82 142 142 142 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 82 82 82 141 141 141 F F F 7 140-159 83 83 83 143 143 143 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 81 81 81 140 140 140 F F F 11 220-239 84 84 84 144 144 144 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 82 82 82 141 141 141 F F F 14 280-299 82 82 82 141 141 141 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 81 81 81 140 140 140 F F F 18 360-379 76 76 76 130 130 130 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 85 85 85 146 146 146 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 83 83 83 143 143 143 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 73 73 73 126 126 126 F F F 25 500-519 84 84 84 144 144 144 F F F 26 520-539 80 80 80 138 138 138 F F F 27 540-559 82 82 82 142 142 142 F F F 28 560-579 85 85 85 146 146 146 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 85, Max flush: 146 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.616201465Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.661016235Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.661642543Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.661687093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.66170225Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.661717056Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528} {"msg":"Crucible c9841411-6f05-4810-8ebc-5b8e42370043 has session id: 57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.662292026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Upstairs opts: Upstairs UUID: c9841411-6f05-4810-8ebc-5b8e42370043, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.662346514Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Crucible stats registered with UUID: c9841411-6f05-4810-8ebc-5b8e42370043","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.662371218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:08.662391954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.662857488Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.662881693Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528} {"msg":"c9841411-6f05-4810-8ebc-5b8e42370043 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.662956276Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.66298079Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"0","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663039217Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"0","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663059752Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"1","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663084797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"1","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.66311326Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"2","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663141384Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"2","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663171757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.66324862Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663284501Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"ds_connection connected from Ok(127.0.0.1:33495)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663317214Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43897)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663351576Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"ds_connection connected from Ok(127.0.0.1:38845)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663441476Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.663984983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"0","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.664024354Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"2","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.664789451Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"2","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.664829571Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"0","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.66536539Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.665482894Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"1","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.665516706Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666006165Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"1","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[0]R flush_numbers[0..12]: [145, 145, 142, 141, 141, 146, 141, 143, 140, 146, 140, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666495424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[0]R generation[0..12]: [84, 84, 82, 82, 82, 85, 82, 83, 81, 85, 81, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666516029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666530896Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[1]R flush_numbers[0..12]: [145, 145, 142, 141, 141, 146, 141, 143, 140, 146, 140, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666545013Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[1]R generation[0..12]: [84, 84, 82, 82, 82, 85, 82, 83, 81, 85, 81, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.66655852Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666572397Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[2]R flush_numbers[0..12]: [145, 145, 142, 141, 141, 146, 141, 143, 140, 146, 140, 144]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666589373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[2]R generation[0..12]: [84, 84, 82, 82, 82, 85, 82, 83, 81, 85, 81, 84]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.66660338Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666617486Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666630274Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666643051Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Next flush: 147","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-01-15T05:22:08.666662476Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666675723Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"":"downstairs","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.66668897Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666700918Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"0","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666713935Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"1","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666726592Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"client":"2","":"downstairs","session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666739589Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666751586Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"c9841411-6f05-4810-8ebc-5b8e42370043 is now active with session: 57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666764103Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666775601Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528,"session_id":"57c0760d-7cb7-4a09-b9dc-b57c6cd6cfdb"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:08.666789658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 8192 data: 8 8 02/30 Write block 53 len 20480 data: 5 5 4 5 5 03/30 Write block 131 len 12288 data: 6 7 6 04/30 Write block 492 len 16384 data: 4 4 7 8 05/30 Write block 368 len 36864 data: 5 4 5 5 4 7 7 7 7 06/30 Write block 370 len 32768 data: 6 6 5 8 8 8 8 9 07/30 Read block 371 len 8192 08/30 Write block 508 len 24576 data: 7 6 6 6 8 10 09/30 Read block 257 len 24576 10/30 Read block 253 len 4096 11/30 Flush 12/30 Write block 357 len 36864 data: 10 9 6 5 6 5 6 8 8 13/30 Flush 14/30 Write block 16 len 4096 data: 6 15/30 Read block 54 len 40960 16/30 Read block 196 len 36864 17/30 Read block 525 len 32768 18/30 Read block 424 len 20480 19/30 Flush 20/30 Write block 360 len 8192 data: 6 7 21/30 Read block 326 len 8192 22/30 Write block 223 len 40960 data: 13 11 12 12 12 12 12 11 10 6 23/30 Read block 400 len 32768 24/30 Read block 581 len 32768 25/30 Read block 75 len 32768 26/30 Write block 527 len 8192 data: 9 9 27/30 Write block 556 len 4096 data: 3 28/30 Write block 281 len 28672 data: 6 5 3 3 3 3 3 29/30 Read block 78 len 32768 30/30 Read block 560 len 4096 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 11 11 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:08.685049212Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1528} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 149 149 149 F F F 1 020-039 84 84 84 145 145 145 F F F 2 040-059 86 86 86 147 147 147 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 86 86 86 147 147 147 F F F 7 140-159 83 83 83 143 143 143 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 81 81 81 140 140 140 F F F 11 220-239 84 84 84 144 144 144 T T T 12 240-259 78 78 78 133 133 133 F F F 13 260-279 82 82 82 141 141 141 F F F 14 280-299 82 82 82 141 141 141 T T T 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 86 86 86 148 148 148 F F F 18 360-379 86 86 86 148 148 148 T T T 19 380-399 85 85 85 146 146 146 F F F 20 400-419 85 85 85 146 146 146 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 83 83 83 143 143 143 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 86 86 86 147 147 147 F F F 25 500-519 86 86 86 147 147 147 F F F 26 520-539 80 80 80 138 138 138 T T T 27 540-559 82 82 82 142 142 142 T T T 28 560-579 85 85 85 146 146 146 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 86, Max flush: 149 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 149 149 149 F F F 1 020-039 84 84 84 145 145 145 F F F 2 040-059 86 86 86 147 147 147 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 86 86 86 147 147 147 F F F 7 140-159 83 83 83 143 143 143 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 81 81 81 140 140 140 F F F 11 220-239 87 87 87 150 150 150 F F F 12 240-259 78 78 78 133 133 133 F F F 13 260-279 82 82 82 141 141 141 F F F 14 280-299 87 87 87 150 150 150 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 86 86 86 148 148 148 F F F 18 360-379 87 87 87 150 150 150 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 85 85 85 146 146 146 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 83 83 83 143 143 143 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 86 86 86 147 147 147 F F F 25 500-519 86 86 86 147 147 147 F F F 26 520-539 87 87 87 150 150 150 F F F 27 540-559 87 87 87 150 150 150 F F F 28 560-579 85 85 85 146 146 146 F F F 29 580-599 72 72 72 125 125 125 F F F Max gen: 87, Max flush: 150 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:12.926673634Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:12.970200226Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:12.970605714Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.003258589Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.003952872Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.003989653Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.004024365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.004040262Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536} {"msg":"Crucible ce017f87-e28d-4761-a7d7-76df439e8406 has session id: 9e1bf68e-1bec-4692-a25e-50a44f4106eb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.004702102Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Upstairs opts: Upstairs UUID: ce017f87-e28d-4761-a7d7-76df439e8406, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.004740193Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Crucible stats registered with UUID: ce017f87-e28d-4761-a7d7-76df439e8406","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.00475551Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:13.004768257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005193191Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005227103Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536} {"msg":"ce017f87-e28d-4761-a7d7-76df439e8406 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005281391Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005311004Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"0","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005349975Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"0","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005379498Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"1","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005410122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"1","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005436945Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"2","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005468938Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"2","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005489173Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005514748Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005541202Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48317)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005568136Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:53115)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.00559361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:36136)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.005618444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.006268237Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"1","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.006312407Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"2","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.006503124Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"0","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.006935886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"1","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.006975807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"2","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007088531Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"0","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007463796Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007538249Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[0]R flush_numbers[0..12]: [149, 145, 147, 141, 141, 146, 147, 143, 140, 146, 140, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007641946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[0]R generation[0..12]: [86, 84, 86, 82, 82, 85, 86, 83, 81, 85, 81, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007662411Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007677368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[1]R flush_numbers[0..12]: [149, 145, 147, 141, 141, 146, 147, 143, 140, 146, 140, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007691385Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[1]R generation[0..12]: [86, 84, 86, 82, 82, 85, 86, 83, 81, 85, 81, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007705482Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007719399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[2]R flush_numbers[0..12]: [149, 145, 147, 141, 141, 146, 147, 143, 140, 146, 140, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007732935Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[2]R generation[0..12]: [86, 84, 86, 82, 82, 85, 86, 83, 81, 85, 81, 87]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007746212Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007759779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007772556Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007785644Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007798061Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007810638Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"":"downstairs","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007832813Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.00784511Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"0","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007858377Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"1","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007871234Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"client":"2","":"downstairs","session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007884071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007896298Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"ce017f87-e28d-4761-a7d7-76df439e8406 is now active with session: 9e1bf68e-1bec-4692-a25e-50a44f4106eb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007908865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.007920483Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536,"session_id":"9e1bf68e-1bec-4692-a25e-50a44f4106eb"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:13.00793254Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 256 len 32768 data: 5 4 5 6 6 7 4 6 03/30 Read block 445 len 24576 04/30 Read block 57 len 20480 05/30 Read block 20 len 40960 06/30 Write block 570 len 24576 data: 8 7 7 6 6 6 07/30 Read block 349 len 4096 08/30 Read block 378 len 28672 09/30 Read block 545 len 4096 10/30 Write block 444 len 32768 data: 11 8 5 6 7 8 10 9 11/30 Write block 260 len 36864 data: 7 8 5 7 8 8 9 9 10 12/30 Write block 231 len 32768 data: 11 7 7 8 10 10 9 10 13/30 Write block 535 len 24576 data: 7 8 5 4 5 4 14/30 Write block 261 len 24576 data: 9 6 8 9 9 10 15/30 Read block 433 len 24576 16/30 Read block 569 len 24576 17/30 Write block 8 len 20480 data: 9 8 6 5 6 18/30 Write block 280 len 32768 data: 6 7 6 4 4 4 4 4 19/30 Read block 319 len 12288 20/30 Write block 581 len 28672 data: 7 8 9 9 9 8 7 21/30 Read block 102 len 32768 22/30 Read block 282 len 36864 23/30 Write block 285 len 4096 data: 5 24/30 Write block 136 len 20480 data: 8 6 7 5 6 25/30 Read block 184 len 8192 26/30 Read block 496 len 28672 27/30 Read block 241 len 32768 28/30 Read block 581 len 16384 29/30 Write block 507 len 36864 data: 6 8 7 7 7 9 11 10 11 30/30 Read block 500 len 8192 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 8 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1000 1000 1000 Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:13.028235542Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1536} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 86 86 86 149 149 149 T T T 1 020-039 84 84 84 145 145 145 F F F 2 040-059 86 86 86 147 147 147 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 86 86 86 147 147 147 T T T 7 140-159 83 83 83 143 143 143 T T T 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 81 81 81 140 140 140 F F F 11 220-239 87 87 87 150 150 150 T T T 12 240-259 78 78 78 133 133 133 T T T 13 260-279 82 82 82 141 141 141 T T T 14 280-299 87 87 87 150 150 150 T T T 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 86 86 86 148 148 148 F F F 18 360-379 87 87 87 150 150 150 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 85 85 85 146 146 146 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 83 83 83 143 143 143 T T T 23 460-479 81 81 81 140 140 140 F F F 24 480-499 86 86 86 147 147 147 F F F 25 500-519 86 86 86 147 147 147 T T T 26 520-539 87 87 87 150 150 150 T T T 27 540-559 87 87 87 150 150 150 T T T 28 560-579 85 85 85 146 146 146 T T T 29 580-599 72 72 72 125 125 125 T T T Max gen: 87, Max flush: 150 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 151 151 151 F F F 1 020-039 84 84 84 145 145 145 F F F 2 040-059 86 86 86 147 147 147 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 85 85 85 146 146 146 F F F 6 120-139 88 88 88 151 151 151 F F F 7 140-159 88 88 88 151 151 151 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 81 81 81 140 140 140 F F F 11 220-239 88 88 88 151 151 151 F F F 12 240-259 88 88 88 151 151 151 F F F 13 260-279 88 88 88 151 151 151 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 86 86 86 148 148 148 F F F 18 360-379 87 87 87 150 150 150 F F F 19 380-399 85 85 85 146 146 146 F F F 20 400-419 85 85 85 146 146 146 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 88 88 88 151 151 151 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 86 86 86 147 147 147 F F F 25 500-519 88 88 88 151 151 151 F F F 26 520-539 88 88 88 151 151 151 F F F 27 540-559 88 88 88 151 151 151 F F F 28 560-579 88 88 88 151 151 151 F F F 29 580-599 88 88 88 151 151 151 F F F Max gen: 88, Max flush: 151 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.339147122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.383782493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.38476776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.384808691Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.384839884Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.38485496Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544} {"msg":"Crucible 5ba30a31-74ce-4d22-ba1c-ece5fd827985 has session id: 3c256780-582a-4b78-8ca4-6e80882f618a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.385442767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Upstairs opts: Upstairs UUID: 5ba30a31-74ce-4d22-ba1c-ece5fd827985, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.385552732Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Crucible stats registered with UUID: 5ba30a31-74ce-4d22-ba1c-ece5fd827985","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.385588794Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:17.385610649Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386049999Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386085631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544} {"msg":"5ba30a31-74ce-4d22-ba1c-ece5fd827985 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386139349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386164324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"0","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386186959Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"0","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386200796Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"1","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386214902Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"1","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386237987Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"2","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386264721Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"2","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386283807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386297714Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.38631198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59141)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386379135Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60668)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386428724Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56046)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.386468725Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.387144732Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"2","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.387173805Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"0","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.387798244Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"0","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.387839014Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"2","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.388437409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.388537736Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"1","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.388570169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389096989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"1","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[0]R flush_numbers[0..12]: [151, 145, 147, 141, 141, 146, 151, 151, 140, 146, 140, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389621941Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[0]R generation[0..12]: [88, 84, 86, 82, 82, 85, 88, 88, 81, 85, 81, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389649464Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.38966869Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[1]R flush_numbers[0..12]: [151, 145, 147, 141, 141, 146, 151, 151, 140, 146, 140, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389686486Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[1]R generation[0..12]: [88, 84, 86, 82, 82, 85, 88, 88, 81, 85, 81, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389708631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389726627Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[2]R flush_numbers[0..12]: [151, 145, 147, 141, 141, 146, 151, 151, 140, 146, 140, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389743713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[2]R generation[0..12]: [88, 84, 86, 82, 82, 85, 88, 88, 81, 85, 81, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389760339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-15T05:22:17.389777495Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Max found gen is 89","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.38980026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Generation requested: 90 >= found:89","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389816286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389833183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389849039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"":"downstairs","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389865186Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389879982Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"0","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389896128Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"1","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389912055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"client":"2","":"downstairs","session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389927961Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389942978Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"5ba30a31-74ce-4d22-ba1c-ece5fd827985 is now active with session: 3c256780-582a-4b78-8ca4-6e80882f618a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389961284Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.38997658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1544,"session_id":"3c256780-582a-4b78-8ca4-6e80882f618a"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:17.389996406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 353 len 36864 data: 11 11 11 12 11 10 7 7 8 02/30 Write block 102 len 8192 data: 9 8 03/30 Read block 195 len 24576 04/30 Read block 328 len 4096 05/30 Read block 29 len 8192 06/30 Read block 301 len 16384 07/30 Read block 546 len 24576 08/30 Read block 65 len 8192 09/30 Write block 211 len 36864 data: 9 9 8 6 5 7 8 9 8 10/30 Read block 404 len 16384 11/30 Read block 174 len 28672 12/30 Read block 71 len 12288 13/30 Read block 395 len 16384 14/30 Flush 15/30 Write block 517 len 32768 data: 10 10 14 14 12 14 12 11 16/30 Flush 17/30 Flush 18/30 Write block 380 len 8192 data: 9 8 19/30 Write block 207 len 40960 data: 8 8 9 7 10 10 9 7 6 8 20/30 Read block 422 len 16384 21/30 Write block 38 len 24576 data: 9 8 9 8 7 10 22/30 Flush 23/30 Write block 577 len 24576 data: 7 5 6 6 8 9 24/30 Write block 126 len 32768 data: 5 5 5 5 7 7 8 7 25/30 Write block 367 len 4096 data: 4 26/30 Write block 131 len 24576 data: 8 9 8 6 6 9 27/30 Read block 122 len 8192 28/30 Read block 500 len 4096 29/30 Read block 279 len 4096 30/30 Write block 587 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:17.40856603Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1544} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 151 151 151 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 90 90 90 152 152 152 F F F 6 120-139 88 88 88 151 151 151 T T T 7 140-159 88 88 88 151 151 151 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 88 88 88 151 151 151 F F F 12 240-259 88 88 88 151 151 151 F F F 13 260-279 88 88 88 151 151 151 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 90 90 90 152 152 152 F F F 18 360-379 90 90 90 152 152 152 T T T 19 380-399 90 90 90 155 155 155 F F F 20 400-419 85 85 85 146 146 146 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 88 88 88 151 151 151 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 86 86 86 147 147 147 F F F 25 500-519 90 90 90 153 153 153 F F F 26 520-539 90 90 90 153 153 153 F F F 27 540-559 88 88 88 151 151 151 F F F 28 560-579 88 88 88 151 151 151 T T T 29 580-599 88 88 88 151 151 151 T T T Max gen: 90, Max flush: 155 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 151 151 151 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 F F F 3 060-079 82 82 82 141 141 141 F F F 4 080-099 82 82 82 141 141 141 F F F 5 100-119 90 90 90 152 152 152 F F F 6 120-139 91 91 91 156 156 156 F F F 7 140-159 88 88 88 151 151 151 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 88 88 88 151 151 151 F F F 12 240-259 88 88 88 151 151 151 F F F 13 260-279 88 88 88 151 151 151 F F F 14 280-299 88 88 88 151 151 151 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 90 90 90 152 152 152 F F F 18 360-379 91 91 91 156 156 156 F F F 19 380-399 90 90 90 155 155 155 F F F 20 400-419 85 85 85 146 146 146 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 88 88 88 151 151 151 F F F 23 460-479 81 81 81 140 140 140 F F F 24 480-499 86 86 86 147 147 147 F F F 25 500-519 90 90 90 153 153 153 F F F 26 520-539 90 90 90 153 153 153 F F F 27 540-559 88 88 88 151 151 151 F F F 28 560-579 91 91 91 156 156 156 F F F 29 580-599 91 91 91 156 156 156 F F F Max gen: 91, Max flush: 156 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.64229444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:21.686811758Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.687300107Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.720272699Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.720861086Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.720902507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.720917094Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.72093133Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552} {"msg":"Crucible e3eb0319-d33b-44cb-9721-686904fccedc has session id: c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.721536653Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Upstairs opts: Upstairs UUID: e3eb0319-d33b-44cb-9721-686904fccedc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.721570666Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Crucible stats registered with UUID: e3eb0319-d33b-44cb-9721-686904fccedc","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.721586622Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:21.721599419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722026642Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722051607Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552} {"msg":"e3eb0319-d33b-44cb-9721-686904fccedc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722120861Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722139077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"0","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722162252Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"0","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722176359Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"1","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722190385Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"1","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722211631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"2","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722237534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"2","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722261779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722287753Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722312358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38929)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722464353Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44572)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722602982Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"ds_connection connected from Ok(127.0.0.1:33850)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.722620368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.723146559Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"0","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.723174572Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"1","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.723189679Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"2","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.723738395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"2","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.723779775Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"1","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.723854808Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"0","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724267365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724355845Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[0]R flush_numbers[0..12]: [151, 155, 155, 141, 141, 152, 156, 151, 140, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.72446558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[0]R generation[0..12]: [88, 90, 90, 82, 82, 90, 91, 88, 81, 85, 90, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724485505Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724500382Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[1]R flush_numbers[0..12]: [151, 155, 155, 141, 141, 152, 156, 151, 140, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724514239Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[1]R generation[0..12]: [88, 90, 90, 82, 82, 90, 91, 88, 81, 85, 90, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724527666Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724541303Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[2]R flush_numbers[0..12]: [151, 155, 155, 141, 141, 152, 156, 151, 140, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.72455486Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[2]R generation[0..12]: [88, 90, 90, 82, 82, 90, 91, 88, 81, 85, 90, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724568267Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724582223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724594921Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724607648Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724620355Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724649508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"":"downstairs","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724674413Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724696038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"0","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724720082Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"1","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724743627Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"client":"2","":"downstairs","session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724764392Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724779479Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"e3eb0319-d33b-44cb-9721-686904fccedc is now active with session: c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724802044Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724827188Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552,"session_id":"c8b8f5fe-994d-47cd-ab68-24d3c8be2dd0"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:21.724847723Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 359 len 40960 data: 8 8 9 6 7 9 9 5 5 6 02/30 Read block 12 len 40960 03/30 Read block 383 len 28672 04/30 Read block 476 len 12288 05/30 Read block 537 len 24576 06/30 Read block 520 len 8192 07/30 Read block 528 len 8192 08/30 Read block 37 len 12288 09/30 Write block 12 len 20480 data: 7 5 4 4 7 10/30 Read block 148 len 28672 11/30 Write block 398 len 20480 data: 8 7 7 9 11 12/30 Flush 13/30 Write block 518 len 36864 data: 11 15 15 13 15 13 12 11 11 14/30 Read block 189 len 28672 15/30 Write block 467 len 4096 data: 8 16/30 Read block 510 len 24576 17/30 Write block 97 len 40960 data: 10 9 7 8 9 10 9 6 7 7 18/30 Read block 135 len 16384 19/30 Write block 403 len 16384 data: 11 11 14 9 20/30 Flush 21/30 Write block 54 len 32768 data: 6 5 6 6 5 5 5 7 22/30 Read block 239 len 40960 23/30 Read block 575 len 16384 24/30 Read block 313 len 16384 25/30 Read block 60 len 40960 26/30 Write block 277 len 24576 data: 4 7 6 7 8 7 27/30 Read block 159 len 8192 28/30 Read block 300 len 28672 29/30 Write block 578 len 16384 data: 6 7 7 9 30/30 Read block 457 len 16384 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 8 10 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:21.746369799Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1552} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 90 90 90 155 155 155 T T T 3 060-079 82 82 82 141 141 141 T T T 4 080-099 92 92 92 158 158 158 F F F 5 100-119 92 92 92 158 158 158 F F F 6 120-139 91 91 91 156 156 156 F F F 7 140-159 88 88 88 151 151 151 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 88 88 88 151 151 151 F F F 12 240-259 88 88 88 151 151 151 F F F 13 260-279 88 88 88 151 151 151 T T T 14 280-299 88 88 88 151 151 151 T T T 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 92 92 92 157 157 157 F F F 18 360-379 92 92 92 157 157 157 F F F 19 380-399 92 92 92 157 157 157 F F F 20 400-419 92 92 92 158 158 158 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 88 88 88 151 151 151 F F F 23 460-479 92 92 92 158 158 158 F F F 24 480-499 86 86 86 147 147 147 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 92 92 92 158 158 158 F F F 27 540-559 88 88 88 151 151 151 F F F 28 560-579 91 91 91 156 156 156 T T T 29 580-599 91 91 91 156 156 156 T T T Max gen: 92, Max flush: 158 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 93 93 93 159 159 159 F F F 3 060-079 93 93 93 159 159 159 F F F 4 080-099 92 92 92 158 158 158 F F F 5 100-119 92 92 92 158 158 158 F F F 6 120-139 91 91 91 156 156 156 F F F 7 140-159 88 88 88 151 151 151 F F F 8 160-179 81 81 81 140 140 140 F F F 9 180-199 85 85 85 146 146 146 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 88 88 88 151 151 151 F F F 12 240-259 88 88 88 151 151 151 F F F 13 260-279 93 93 93 159 159 159 F F F 14 280-299 93 93 93 159 159 159 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 78 78 78 133 133 133 F F F 17 340-359 92 92 92 157 157 157 F F F 18 360-379 92 92 92 157 157 157 F F F 19 380-399 92 92 92 157 157 157 F F F 20 400-419 92 92 92 158 158 158 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 88 88 88 151 151 151 F F F 23 460-479 92 92 92 158 158 158 F F F 24 480-499 86 86 86 147 147 147 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 92 92 92 158 158 158 F F F 27 540-559 88 88 88 151 151 151 F F F 28 560-579 93 93 93 159 159 159 F F F 29 580-599 93 93 93 159 159 159 F F F Max gen: 93, Max flush: 159 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.049922815Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.094873534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.095712584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.095754495Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.095769451Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.095783798Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560} {"msg":"Crucible 17e77f8d-3eea-4a9b-a29c-eeb86c69fe7a has session id: 3cf3452e-3af7-4bf2-8702-584719a58462","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.096418854Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Upstairs opts: Upstairs UUID: 17e77f8d-3eea-4a9b-a29c-eeb86c69fe7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.096466624Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Crucible stats registered with UUID: 17e77f8d-3eea-4a9b-a29c-eeb86c69fe7a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.096492498Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:26.096514503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.096898126Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.096928249Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560} {"msg":"17e77f8d-3eea-4a9b-a29c-eeb86c69fe7a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.096994394Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.09701176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"0","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097035875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"0","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097053091Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"1","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097078355Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"1","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.09710164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"2","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097127814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"2","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097153648Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097178332Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097199427Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"ds_connection connected from Ok(127.0.0.1:54616)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097223992Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"ds_connection connected from Ok(127.0.0.1:52359)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097292886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"ds_connection connected from Ok(127.0.0.1:45791)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097337076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.097976962Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"0","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.098009624Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"1","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.098676513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"0","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.098796126Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"1","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.099233467Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.099412796Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.0994395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"2","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.099960082Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"2","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[0]R flush_numbers[0..12]: [157, 155, 159, 159, 158, 158, 156, 151, 140, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100447332Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[0]R generation[0..12]: [92, 90, 93, 93, 92, 92, 91, 88, 81, 85, 90, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100471826Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100486673Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[1]R flush_numbers[0..12]: [157, 155, 159, 159, 158, 158, 156, 151, 140, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.10050076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[1]R generation[0..12]: [92, 90, 93, 93, 92, 92, 91, 88, 81, 85, 90, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100514487Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100528473Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[2]R flush_numbers[0..12]: [157, 155, 159, 159, 158, 158, 156, 151, 140, 146, 155, 151]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100542241Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[2]R generation[0..12]: [92, 90, 93, 93, 92, 92, 91, 88, 81, 85, 90, 88]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100561056Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100575303Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.10058781Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:22:26.100600727Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100618803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.10063132Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"":"downstairs","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100644207Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100655935Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"0","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100668962Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"1","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100682509Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"client":"2","":"downstairs","session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100695556Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100707533Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"17e77f8d-3eea-4a9b-a29c-eeb86c69fe7a is now active with session: 3cf3452e-3af7-4bf2-8702-584719a58462","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.10071997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100731607Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560,"session_id":"3cf3452e-3af7-4bf2-8702-584719a58462"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:26.100743395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 396 len 32768 data: 7 6 9 8 8 10 12 12 02/30 Write block 187 len 36864 data: 8 7 8 10 11 11 8 9 8 03/30 Write block 43 len 24576 data: 11 10 11 8 8 8 04/30 Flush 05/30 Flush 06/30 Write block 356 len 8192 data: 13 12 07/30 Read block 246 len 36864 08/30 Read block 36 len 36864 09/30 Read block 139 len 20480 10/30 Read block 67 len 40960 11/30 Write block 320 len 28672 data: 9 12 10 10 7 6 5 12/30 Write block 138 len 40960 data: 8 6 7 6 8 8 8 10 12 10 13/30 Write block 44 len 12288 data: 11 12 9 14/30 Read block 234 len 16384 15/30 Read block 65 len 4096 16/30 Write block 582 len 12288 data: 10 10 10 17/30 Write block 164 len 8192 data: 8 8 18/30 Write block 479 len 12288 data: 5 5 6 19/30 Read block 44 len 16384 20/30 Write block 289 len 8192 data: 4 4 21/30 Write block 237 len 28672 data: 10 11 7 8 11 11 9 22/30 Read block 457 len 32768 23/30 Write block 95 len 32768 data: 11 10 11 10 8 9 10 11 24/30 Write block 415 len 4096 data: 7 25/30 Flush 26/30 Write block 445 len 40960 data: 9 6 7 8 9 11 10 9 7 7 27/30 Read block 231 len 40960 28/30 Read block 58 len 28672 29/30 Write block 402 len 12288 data: 13 13 12 30/30 Write block 476 len 24576 data: 6 4 3 6 6 7 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:26.120206647Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1560} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 94 94 94 162 162 162 F F F 3 060-079 93 93 93 159 159 159 F F F 4 080-099 94 94 94 162 162 162 F F F 5 100-119 94 94 94 162 162 162 F F F 6 120-139 94 94 94 162 162 162 F F F 7 140-159 94 94 94 162 162 162 F F F 8 160-179 94 94 94 162 162 162 F F F 9 180-199 94 94 94 160 160 160 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 94 94 94 162 162 162 F F F 12 240-259 94 94 94 162 162 162 F F F 13 260-279 93 93 93 159 159 159 F F F 14 280-299 94 94 94 162 162 162 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 92 92 92 157 157 157 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 94 94 94 162 162 162 T T T 21 420-439 84 84 84 145 145 145 F F F 22 440-459 88 88 88 151 151 151 T T T 23 460-479 94 94 94 162 162 162 T T T 24 480-499 94 94 94 162 162 162 T T T 25 500-519 92 92 92 158 158 158 F F F 26 520-539 92 92 92 158 158 158 F F F 27 540-559 88 88 88 151 151 151 F F F 28 560-579 93 93 93 159 159 159 F F F 29 580-599 94 94 94 162 162 162 F F F Max gen: 94, Max flush: 162 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 90 90 90 155 155 155 F F F 2 040-059 94 94 94 162 162 162 F F F 3 060-079 93 93 93 159 159 159 F F F 4 080-099 94 94 94 162 162 162 F F F 5 100-119 94 94 94 162 162 162 F F F 6 120-139 94 94 94 162 162 162 F F F 7 140-159 94 94 94 162 162 162 F F F 8 160-179 94 94 94 162 162 162 F F F 9 180-199 94 94 94 160 160 160 F F F 10 200-219 90 90 90 155 155 155 F F F 11 220-239 94 94 94 162 162 162 F F F 12 240-259 94 94 94 162 162 162 F F F 13 260-279 93 93 93 159 159 159 F F F 14 280-299 94 94 94 162 162 162 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 92 92 92 157 157 157 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 95 95 95 163 163 163 F F F 23 460-479 95 95 95 163 163 163 F F F 24 480-499 95 95 95 163 163 163 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 92 92 92 158 158 158 F F F 27 540-559 88 88 88 151 151 151 F F F 28 560-579 93 93 93 159 159 159 F F F 29 580-599 94 94 94 162 162 162 F F F Max gen: 95, Max flush: 163 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.355481486Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.400152079Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.40094115Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.401008415Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.401034779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.401063113Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568} {"msg":"Crucible be7a76f8-2eb8-4f9d-a4a7-bfc4b2fefffb has session id: dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.401770812Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Upstairs opts: Upstairs UUID: be7a76f8-2eb8-4f9d-a4a7-bfc4b2fefffb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.401822321Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Crucible stats registered with UUID: be7a76f8-2eb8-4f9d-a4a7-bfc4b2fefffb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.401846535Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:30.401867011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402260742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402287875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568} {"msg":"be7a76f8-2eb8-4f9d-a4a7-bfc4b2fefffb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402351061Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402370387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"0","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402395491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"0","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402409428Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"1","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402423345Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"1","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402437971Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"2","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402462306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"2","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402485631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402509185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402536449Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54847)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402646284Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55375)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402825314Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61828)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.402863615Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.403410951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"0","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.403441614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"1","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.403588701Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"2","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404068672Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"0","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404106763Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"1","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404222797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"2","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.40465005Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404680043Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[0]R flush_numbers[0..12]: [157, 155, 162, 159, 162, 162, 162, 162, 162, 160, 155, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404713876Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[0]R generation[0..12]: [92, 90, 94, 93, 94, 94, 94, 94, 94, 94, 90, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404730972Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404745369Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[1]R flush_numbers[0..12]: [157, 155, 162, 159, 162, 162, 162, 162, 162, 160, 155, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404762815Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[1]R generation[0..12]: [92, 90, 94, 93, 94, 94, 94, 94, 94, 94, 90, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404776352Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404790278Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[2]R flush_numbers[0..12]: [157, 155, 162, 159, 162, 162, 162, 162, 162, 160, 155, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404803626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[2]R generation[0..12]: [92, 90, 94, 93, 94, 94, 94, 94, 94, 94, 90, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404816533Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404830349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404842997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404855794Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404868321Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404880598Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"":"downstairs","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404893675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404913091Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"0","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404936525Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"1","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.4049579Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"client":"2","":"downstairs","session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.404985374Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.405000331Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"be7a76f8-2eb8-4f9d-a4a7-bfc4b2fefffb is now active with session: dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.405013158Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.405024975Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568,"session_id":"dbec0d6e-7fa1-4024-b92f-fe27b2ee8df9"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:30.405036953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 24576 data: 8 7 5 5 5 8 02/30 Read block 211 len 16384 03/30 Flush 04/30 Write block 117 len 32768 data: 11 10 8 8 6 6 6 5 05/30 Read block 128 len 20480 06/30 Read block 266 len 40960 07/30 Read block 450 len 36864 08/30 Read block 107 len 24576 09/30 Write block 72 len 20480 data: 8 7 7 4 3 10/30 Read block 529 len 28672 11/30 Read block 418 len 4096 12/30 Read block 533 len 8192 13/30 Write block 473 len 28672 data: 8 7 6 7 5 4 7 14/30 Read block 451 len 40960 15/30 Flush 16/30 Write block 447 len 4096 data: 8 17/30 Write block 207 len 12288 data: 9 9 10 18/30 Write block 69 len 16384 data: 12 13 12 9 19/30 Read block 40 len 8192 20/30 Read block 192 len 32768 21/30 Write block 262 len 40960 data: 7 9 10 10 11 10 11 8 6 5 22/30 Write block 153 len 4096 data: 4 23/30 Write block 102 len 8192 data: 12 10 24/30 Write block 536 len 24576 data: 9 6 5 6 5 6 25/30 Write block 29 len 40960 data: 6 6 6 7 7 6 6 6 7 10 26/30 Read block 329 len 24576 27/30 Read block 459 len 40960 28/30 Write block 50 len 24576 data: 5 5 5 6 7 6 29/30 Write block 166 len 12288 data: 8 8 7 30/30 Write block 377 len 8192 data: 10 9 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 1 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:30.424149114Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1568} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 90 90 90 155 155 155 T T T 2 040-059 94 94 94 162 162 162 T T T 3 060-079 96 96 96 165 165 165 T T T 4 080-099 94 94 94 162 162 162 F F F 5 100-119 96 96 96 165 165 165 T T T 6 120-139 96 96 96 165 165 165 F F F 7 140-159 94 94 94 162 162 162 T T T 8 160-179 94 94 94 162 162 162 T T T 9 180-199 94 94 94 160 160 160 F F F 10 200-219 90 90 90 155 155 155 T T T 11 220-239 94 94 94 162 162 162 F F F 12 240-259 94 94 94 162 162 162 F F F 13 260-279 93 93 93 159 159 159 T T T 14 280-299 94 94 94 162 162 162 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 92 92 92 157 157 157 T T T 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 95 95 95 163 163 163 T T T 23 460-479 96 96 96 165 165 165 F F F 24 480-499 96 96 96 164 164 164 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 92 92 92 158 158 158 T T T 27 540-559 88 88 88 151 151 151 T T T 28 560-579 93 93 93 159 159 159 F F F 29 580-599 94 94 94 162 162 162 F F F Max gen: 96, Max flush: 165 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 97 97 97 166 166 166 F F F 2 040-059 97 97 97 166 166 166 F F F 3 060-079 97 97 97 166 166 166 F F F 4 080-099 94 94 94 162 162 162 F F F 5 100-119 97 97 97 166 166 166 F F F 6 120-139 96 96 96 165 165 165 F F F 7 140-159 97 97 97 166 166 166 F F F 8 160-179 97 97 97 166 166 166 F F F 9 180-199 94 94 94 160 160 160 F F F 10 200-219 97 97 97 166 166 166 F F F 11 220-239 94 94 94 162 162 162 F F F 12 240-259 94 94 94 162 162 162 F F F 13 260-279 97 97 97 166 166 166 F F F 14 280-299 94 94 94 162 162 162 F F F 15 300-319 84 84 84 144 144 144 F F F 16 320-339 94 94 94 162 162 162 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 97 97 97 166 166 166 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 84 84 84 145 145 145 F F F 22 440-459 97 97 97 166 166 166 F F F 23 460-479 96 96 96 165 165 165 F F F 24 480-499 96 96 96 164 164 164 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 97 97 97 166 166 166 F F F 27 540-559 97 97 97 166 166 166 F F F 28 560-579 93 93 93 159 159 159 F F F 29 580-599 94 94 94 162 162 162 F F F Max gen: 97, Max flush: 166 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.704379632Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.74942392Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.750209362Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.75026169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.750281296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.750299852Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576} {"msg":"Crucible de9bae44-1ec8-4068-ab48-2fe408e3a24c has session id: c4bdbbae-bda7-48c6-ad0e-2b568002be3f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.750918692Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Upstairs opts: Upstairs UUID: de9bae44-1ec8-4068-ab48-2fe408e3a24c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.750963311Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Crucible stats registered with UUID: de9bae44-1ec8-4068-ab48-2fe408e3a24c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.750986466Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:34.751006542Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751559436Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751584751Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576} {"msg":"de9bae44-1ec8-4068-ab48-2fe408e3a24c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751638769Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751670851Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"0","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751699375Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"0","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751717731Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"1","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751732307Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"1","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751747624Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"2","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751773538Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"2","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751800742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751829915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751857649Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52975)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751885833Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49766)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.751916895Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40708)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.75216262Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.752728802Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"2","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.752766933Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"0","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.752853933Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"1","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.753299922Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"0","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.753403569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"1","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.753432892Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"2","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.753829882Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.753916553Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[0]R flush_numbers[0..12]: [157, 166, 166, 166, 162, 166, 165, 166, 166, 160, 166, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.753999304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[0]R generation[0..12]: [92, 97, 97, 97, 94, 97, 96, 97, 97, 94, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754019869Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754034296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[1]R flush_numbers[0..12]: [157, 166, 166, 166, 162, 166, 165, 166, 166, 160, 166, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754048953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[1]R generation[0..12]: [92, 97, 97, 97, 94, 97, 96, 97, 97, 94, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.75406284Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754076866Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[2]R flush_numbers[0..12]: [157, 166, 166, 166, 162, 166, 165, 166, 166, 160, 166, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754090553Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[2]R generation[0..12]: [92, 97, 97, 97, 94, 97, 96, 97, 97, 94, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.75410441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754118217Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754130894Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.75414823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754161257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754173814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"":"downstairs","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"No downstairs reconciliation required","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.75419363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754205847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"0","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754218944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"1","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754231621Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"client":"2","":"downstairs","session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754244478Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754256606Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"de9bae44-1ec8-4068-ab48-2fe408e3a24c is now active with session: c4bdbbae-bda7-48c6-ad0e-2b568002be3f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754269313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.75428092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576,"session_id":"c4bdbbae-bda7-48c6-ad0e-2b568002be3f"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:34.754292708Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 316 len 36864 data: 7 8 8 9 10 13 11 11 8 02/30 Flush 03/30 Flush 04/30 Write block 96 len 28672 data: 11 12 11 9 10 11 13 05/30 Write block 255 len 28672 data: 4 6 5 6 7 8 10 06/30 Read block 516 len 16384 07/30 Write block 432 len 32768 data: 9 9 10 8 7 8 10 9 08/30 Read block 217 len 24576 09/30 Write block 261 len 32768 data: 11 8 10 11 11 12 11 12 10/30 Flush 11/30 Write block 41 len 20480 data: 9 8 12 12 13 12/30 Read block 311 len 28672 13/30 Flush 14/30 Write block 286 len 20480 data: 5 5 3 5 5 15/30 Read block 294 len 32768 16/30 Read block 58 len 40960 17/30 Write block 373 len 24576 data: 9 9 9 9 11 10 18/30 Read block 300 len 20480 19/30 Read block 110 len 8192 20/30 Read block 182 len 24576 21/30 Write block 281 len 28672 data: 9 8 5 5 6 6 6 22/30 Flush 23/30 Write block 587 len 20480 data: 9 6 6 6 5 24/30 Read block 582 len 32768 25/30 Write block 186 len 36864 data: 8 9 8 9 11 12 12 9 10 26/30 Write block 575 len 20480 data: 7 8 8 7 8 27/30 Read block 101 len 4096 28/30 Read block 9 len 40960 29/30 Read block 62 len 4096 30/30 Write block 534 len 8192 data: 7 8 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:34.77494112Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1576} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 97 97 97 166 166 166 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 97 97 97 166 166 166 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 98 98 98 169 169 169 F F F 6 120-139 96 96 96 165 165 165 F F F 7 140-159 97 97 97 166 166 166 F F F 8 160-179 97 97 97 166 166 166 F F F 9 180-199 94 94 94 160 160 160 T T T 10 200-219 97 97 97 166 166 166 F F F 11 220-239 94 94 94 162 162 162 F F F 12 240-259 98 98 98 169 169 169 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 98 98 98 167 167 167 F F F 16 320-339 98 98 98 167 167 167 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 98 98 98 171 171 171 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 97 97 97 166 166 166 F F F 23 460-479 96 96 96 165 165 165 F F F 24 480-499 96 96 96 164 164 164 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 97 97 97 166 166 166 T T T 27 540-559 97 97 97 166 166 166 F F F 28 560-579 93 93 93 159 159 159 T T T 29 580-599 94 94 94 162 162 162 T T T Max gen: 98, Max flush: 171 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 97 97 97 166 166 166 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 97 97 97 166 166 166 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 98 98 98 169 169 169 F F F 6 120-139 96 96 96 165 165 165 F F F 7 140-159 97 97 97 166 166 166 F F F 8 160-179 97 97 97 166 166 166 F F F 9 180-199 99 99 99 172 172 172 F F F 10 200-219 97 97 97 166 166 166 F F F 11 220-239 94 94 94 162 162 162 F F F 12 240-259 98 98 98 169 169 169 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 98 98 98 167 167 167 F F F 16 320-339 98 98 98 167 167 167 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 98 98 98 171 171 171 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 95 95 95 163 163 163 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 97 97 97 166 166 166 F F F 23 460-479 96 96 96 165 165 165 F F F 24 480-499 96 96 96 164 164 164 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 99 99 99 172 172 172 F F F 27 540-559 97 97 97 166 166 166 F F F 28 560-579 99 99 99 172 172 172 F F F 29 580-599 99 99 99 172 172 172 F F F Max gen: 99, Max flush: 172 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.016129991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.061179388Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.061954313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.061999743Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.062023657Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.062039954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584} {"msg":"Crucible 09b5512f-9764-4e95-afee-fb439ecf5543 has session id: f1decaa8-fe91-41fb-a845-81b597d96a3d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.062588569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Upstairs opts: Upstairs UUID: 09b5512f-9764-4e95-afee-fb439ecf5543, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.062642587Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Crucible stats registered with UUID: 09b5512f-9764-4e95-afee-fb439ecf5543","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.062666682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:39.062691966Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063085607Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063112351Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584} {"msg":"09b5512f-9764-4e95-afee-fb439ecf5543 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063180455Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063209719Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"0","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.06324829Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"0","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063275784Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"1","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063305207Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"1","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063332661Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"2","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063376901Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"2","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063407934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063436617Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063460052Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62414)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063484566Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48693)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063597161Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43361)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.063698538Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.064228888Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"0","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.064270208Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"1","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.064758808Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"2","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.064884089Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"0","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.064906384Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"1","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065286658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"2","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065442953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065529463Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[0]R flush_numbers[0..12]: [157, 166, 170, 166, 169, 169, 165, 166, 166, 172, 166, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065788964Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[0]R generation[0..12]: [92, 97, 98, 97, 98, 98, 96, 97, 97, 99, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065812099Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065826606Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[1]R flush_numbers[0..12]: [157, 166, 170, 166, 169, 169, 165, 166, 166, 172, 166, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065840983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[1]R generation[0..12]: [92, 97, 98, 97, 98, 98, 96, 97, 97, 99, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.06585486Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065869046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[2]R flush_numbers[0..12]: [157, 166, 170, 166, 169, 169, 165, 166, 166, 172, 166, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065886513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[2]R generation[0..12]: [92, 97, 98, 97, 98, 98, 96, 97, 97, 99, 97, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065900419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065914036Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065926953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.06594011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs",""Wait for a query_work_queue command to finish before sending IO :"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065960286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065973223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"":"downstairs","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.06598625Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.065998117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"0","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.066011164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"1","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.066024031Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"client":"2","":"downstairs","session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.066036928Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.066048986Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"09b5512f-9764-4e95-afee-fb439ecf5543 is now active with session: f1decaa8-fe91-41fb-a845-81b597d96a3d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.066061733Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-15T05:22:39.06607357Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584,"session_id":"f1decaa8-fe91-41fb-a845-81b597d96a3d"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:39.066091716Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 472 len 20480 data: 10 9 8 7 8 02/30 Flush 03/30 Write block 445 len 12288 data: 10 7 9 04/30 Write block 400 len 16384 data: 9 11 14 14 05/30 Read block 113 len 4096 06/30 Read block 44 len 28672 07/30 Read block 254 len 24576 08/30 Read block 283 len 20480 09/30 Read block 387 len 8192 10/30 Read block 195 len 8192 11/30 Read block 538 len 20480 12/30 Flush 13/30 Write block 243 len 40960 data: 10 7 7 8 6 8 8 6 7 5 14/30 Read block 392 len 4096 15/30 Read block 504 len 4096 16/30 Write block 188 len 8192 data: 9 10 17/30 Write block 144 len 20480 data: 9 11 13 11 10 18/30 Write block 533 len 24576 data: 5 8 9 10 7 6 19/30 Flush 20/30 Write block 326 len 40960 data: 6 6 6 6 6 5 5 6 7 9 21/30 Write block 579 len 20480 data: 9 8 10 11 11 22/30 Read block 490 len 24576 23/30 Write block 221 len 28672 data: 8 11 14 12 13 13 13 24/30 Write block 524 len 20480 data: 13 12 12 10 10 25/30 Write block 223 len 32768 data: 15 13 14 14 14 13 13 12 26/30 Read block 246 len 4096 27/30 Write block 372 len 24576 data: 6 10 10 10 10 12 28/30 Read block 57 len 32768 29/30 Write block 232 len 12288 data: 8 8 9 30/30 Write block 225 len 32768 data: 15 15 15 14 14 13 12 9 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 10 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:39.084639765Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1584} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 97 97 97 166 166 166 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 97 97 97 166 166 166 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 98 98 98 169 169 169 F F F 6 120-139 96 96 96 165 165 165 F F F 7 140-159 100 100 100 175 175 175 F F F 8 160-179 97 97 97 166 166 166 F F F 9 180-199 100 100 100 175 175 175 F F F 10 200-219 97 97 97 166 166 166 F F F 11 220-239 94 94 94 162 162 162 T T T 12 240-259 100 100 100 175 175 175 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 98 98 98 167 167 167 F F F 16 320-339 98 98 98 167 167 167 T T T 17 340-359 94 94 94 162 162 162 F F F 18 360-379 98 98 98 171 171 171 T T T 19 380-399 94 94 94 160 160 160 F F F 20 400-419 100 100 100 174 174 174 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 100 100 100 174 174 174 F F F 23 460-479 100 100 100 173 173 173 F F F 24 480-499 96 96 96 164 164 164 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 100 100 100 175 175 175 T T T 27 540-559 97 97 97 166 166 166 F F F 28 560-579 99 99 99 172 172 172 T T T 29 580-599 99 99 99 172 172 172 T T T Max gen: 100, Max flush: 175 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 97 97 97 166 166 166 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 97 97 97 166 166 166 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 98 98 98 169 169 169 F F F 6 120-139 96 96 96 165 165 165 F F F 7 140-159 100 100 100 175 175 175 F F F 8 160-179 97 97 97 166 166 166 F F F 9 180-199 100 100 100 175 175 175 F F F 10 200-219 97 97 97 166 166 166 F F F 11 220-239 101 101 101 176 176 176 F F F 12 240-259 100 100 100 175 175 175 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 98 98 98 167 167 167 F F F 16 320-339 101 101 101 176 176 176 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 101 101 101 176 176 176 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 100 100 100 174 174 174 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 100 100 100 174 174 174 F F F 23 460-479 100 100 100 173 173 173 F F F 24 480-499 96 96 96 164 164 164 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 101 101 101 176 176 176 F F F 27 540-559 97 97 97 166 166 166 F F F 28 560-579 101 101 101 176 176 176 F F F 29 580-599 101 101 101 176 176 176 F F F Max gen: 101, Max flush: 176 Begin loop 51 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.333736205Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.378944916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.379603606Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.379657394Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.379680719Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.379698875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592} {"msg":"Crucible fa771a8f-8ad3-46df-9ced-bc1cc5435288 has session id: 479ff858-d1f7-4ebb-8ac8-1c70414477e4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380274305Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Upstairs opts: Upstairs UUID: fa771a8f-8ad3-46df-9ced-bc1cc5435288, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380321194Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Crucible stats registered with UUID: fa771a8f-8ad3-46df-9ced-bc1cc5435288","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380345169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:43.380365194Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380760155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380793247Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592} {"msg":"fa771a8f-8ad3-46df-9ced-bc1cc5435288 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380827109Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380843406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"0","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380871889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"0","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380888986Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"1","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380906881Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"1","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380930906Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"2","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380961429Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"2","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.380989573Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.381015607Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.381042421Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65516)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.381068795Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:45746)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.381094379Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:48040)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.381120943Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.381788152Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"0","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.381815216Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"1","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.382423758Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"1","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.382495312Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"0","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.382935122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.383072611Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.383327183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"2","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.383820391Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"2","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[0]R flush_numbers[0..12]: [157, 166, 170, 166, 169, 169, 165, 175, 166, 175, 166, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.38431384Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[0]R generation[0..12]: [92, 97, 98, 97, 98, 98, 96, 100, 97, 100, 97, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384337554Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384352191Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[1]R flush_numbers[0..12]: [157, 166, 170, 166, 169, 169, 165, 175, 166, 175, 166, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384366318Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[1]R generation[0..12]: [92, 97, 98, 97, 98, 98, 96, 100, 97, 100, 97, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384380005Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384393442Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[2]R flush_numbers[0..12]: [157, 166, 170, 166, 169, 169, 165, 175, 166, 175, 166, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384407109Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[2]R generation[0..12]: [92, 97, 98, 97, 98, 98, 96, 100, 97, 100, 97, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384420376Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384433932Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.38444672Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384459617Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384478042Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.38449054Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"":"downstairs","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384503257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384518683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"0","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384532Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"1","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384544847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"client":"2","":"downstairs","session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384557995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384570022Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"fa771a8f-8ad3-46df-9ced-bc1cc5435288 is now active with session: 479ff858-d1f7-4ebb-8ac8-1c70414477e4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384582449Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384593976Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592,"session_id":"479ff858-d1f7-4ebb-8ac8-1c70414477e4"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:43.384605874Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 12288 data: 10 9 10 02/30 Read block 37 len 4096 03/30 Flush 04/30 Write block 592 len 8192 data: 4 3 05/30 Read block 516 len 4096 06/30 Write block 532 len 12288 data: 5 6 9 07/30 Read block 99 len 28672 08/30 Write block 66 len 8192 data: 8 9 09/30 Write block 21 len 4096 data: 8 10/30 Flush 11/30 Write block 171 len 8192 data: 8 10 12/30 Read block 529 len 24576 13/30 Write block 459 len 16384 data: 6 4 4 5 14/30 Read block 236 len 8192 15/30 Read block 331 len 8192 16/30 Read block 103 len 4096 17/30 Write block 571 len 36864 data: 8 8 7 7 8 9 9 8 10 18/30 Flush 19/30 Write block 486 len 28672 data: 9 11 11 9 8 6 6 20/30 Read block 519 len 16384 21/30 Write block 572 len 8192 data: 9 8 22/30 Read block 76 len 24576 23/30 Flush 24/30 Write block 486 len 16384 data: 10 12 12 10 25/30 Read block 44 len 4096 26/30 Read block 129 len 8192 27/30 Write block 571 len 40960 data: 9 10 9 8 9 10 10 9 11 9 28/30 Write block 170 len 8192 data: 8 9 29/30 Write block 469 len 36864 data: 7 8 9 11 10 9 8 9 6 30/30 Write block 223 len 36864 data: 16 14 16 16 16 15 15 14 13 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:43.402177574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1592} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 102 102 102 178 178 178 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 102 102 102 178 178 178 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 98 98 98 169 169 169 F F F 6 120-139 96 96 96 165 165 165 F F F 7 140-159 100 100 100 175 175 175 F F F 8 160-179 102 102 102 179 179 179 T T T 9 180-199 100 100 100 175 175 175 F F F 10 200-219 97 97 97 166 166 166 F F F 11 220-239 102 102 102 177 177 177 T T T 12 240-259 100 100 100 175 175 175 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 98 98 98 167 167 167 F F F 16 320-339 101 101 101 176 176 176 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 101 101 101 176 176 176 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 100 100 100 174 174 174 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 102 102 102 179 179 179 F F F 23 460-479 102 102 102 179 179 179 T T T 24 480-499 102 102 102 180 180 180 T T T 25 500-519 92 92 92 158 158 158 F F F 26 520-539 102 102 102 178 178 178 F F F 27 540-559 97 97 97 166 166 166 F F F 28 560-579 102 102 102 180 180 180 T T T 29 580-599 102 102 102 178 178 178 T T T Max gen: 102, Max flush: 180 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 102 102 102 178 178 178 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 102 102 102 178 178 178 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 98 98 98 169 169 169 F F F 6 120-139 96 96 96 165 165 165 F F F 7 140-159 100 100 100 175 175 175 F F F 8 160-179 103 103 103 181 181 181 F F F 9 180-199 100 100 100 175 175 175 F F F 10 200-219 97 97 97 166 166 166 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 100 100 100 175 175 175 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 98 98 98 167 167 167 F F F 16 320-339 101 101 101 176 176 176 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 101 101 101 176 176 176 F F F 19 380-399 94 94 94 160 160 160 F F F 20 400-419 100 100 100 174 174 174 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 102 102 102 179 179 179 F F F 23 460-479 103 103 103 181 181 181 F F F 24 480-499 103 103 103 181 181 181 F F F 25 500-519 92 92 92 158 158 158 F F F 26 520-539 102 102 102 178 178 178 F F F 27 540-559 97 97 97 166 166 166 F F F 28 560-579 103 103 103 181 181 181 F F F 29 580-599 103 103 103 181 181 181 F F F Max gen: 103, Max flush: 181 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.647947437Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.693151339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.693881833Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.693925064Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.69393985Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.693954447Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600} {"msg":"Crucible 476ee539-2cb4-41dc-8968-fb30d6aa4cba has session id: 7ac6557e-825f-47cd-962c-b46e30eaa313","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.694525937Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Upstairs opts: Upstairs UUID: 476ee539-2cb4-41dc-8968-fb30d6aa4cba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.694571047Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Crucible stats registered with UUID: 476ee539-2cb4-41dc-8968-fb30d6aa4cba","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.694597711Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:47.694619056Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695046679Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695072454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600} {"msg":"476ee539-2cb4-41dc-8968-fb30d6aa4cba active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695144777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695163213Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"0","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695191157Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"0","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695215051Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"1","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695231568Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"1","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695244435Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"2","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695267859Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"2","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695292024Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695315658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695361488Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"ds_connection connected from Ok(127.0.0.1:37764)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.69557301Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"ds_connection connected from Ok(127.0.0.1:44722)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695602933Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"ds_connection connected from Ok(127.0.0.1:35748)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.695624759Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.69619327Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"2","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.696222273Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"0","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.69663567Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"1","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.696851721Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"0","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.696954857Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"2","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697237613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"1","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697419602Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697455404Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[0]R flush_numbers[0..12]: [157, 178, 170, 178, 169, 169, 165, 175, 181, 175, 166, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697768873Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[0]R generation[0..12]: [92, 102, 98, 102, 98, 98, 96, 100, 103, 100, 97, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697797866Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697813903Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[1]R flush_numbers[0..12]: [157, 178, 170, 178, 169, 169, 165, 175, 181, 175, 166, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.69782798Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[1]R generation[0..12]: [92, 102, 98, 102, 98, 98, 96, 100, 103, 100, 97, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697841917Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697856013Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[2]R flush_numbers[0..12]: [157, 178, 170, 178, 169, 169, 165, 175, 181, 175, 166, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.69786961Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[2]R generation[0..12]: [92, 102, 98, 102, 98, 98, 96, 100, 103, 100, 97, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697883157Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697896984Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.69791444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697928467Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E"Wait for a query_work_queue command to finish before sending IO ,"pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697947482Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.69796009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"":"downstairs","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697973157Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697985254Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"0","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.697998271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"1","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.698011058Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"client":"2","":"downstairs","session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.698024155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.698036202Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"476ee539-2cb4-41dc-8968-fb30d6aa4cba is now active with session: 7ac6557e-825f-47cd-962c-b46e30eaa313","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.698048759Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.698060457Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600,"session_id":"7ac6557e-825f-47cd-962c-b46e30eaa313"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:47.698072374Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 32768 data: 7 7 8 8 10 10 9 9 02/30 Flush 03/30 Write block 324 len 24576 data: 9 7 7 7 7 7 04/30 Read block 427 len 16384 05/30 Write block 157 len 8192 data: 12 10 06/30 Read block 559 len 32768 07/30 Read block 482 len 16384 08/30 Write block 301 len 28672 data: 2 4 6 7 4 4 5 09/30 Read block 354 len 36864 10/30 Read block 569 len 24576 11/30 Flush 12/30 Write block 463 len 28672 data: 6 7 7 9 9 10 8 13/30 Write block 469 len 4096 data: 9 14/30 Read block 198 len 32768 15/30 Flush 16/30 Write block 112 len 36864 data: 6 7 7 10 12 12 11 9 9 17/30 Read block 159 len 16384 18/30 Read block 474 len 28672 19/30 Write block 183 len 28672 data: 10 8 7 9 10 10 11 20/30 Read block 512 len 4096 21/30 Write block 215 len 4096 data: 7 22/30 Write block 375 len 24576 data: 11 11 13 11 11 10 23/30 Read block 182 len 16384 24/30 Flush 25/30 Write block 322 len 12288 data: 12 12 10 26/30 Read block 527 len 36864 27/30 Read block 312 len 8192 28/30 Write block 410 len 36864 data: 7 9 7 7 7 8 7 6 5 29/30 Write block 386 len 32768 data: 9 9 10 9 10 8 9 9 30/30 Read block 178 len 16384 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 5 6 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:47.71840047Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1600} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 102 102 102 178 178 178 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 102 102 102 178 178 178 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 104 104 104 185 185 185 F F F 6 120-139 104 104 104 185 185 185 F F F 7 140-159 104 104 104 183 183 183 F F F 8 160-179 103 103 103 181 181 181 F F F 9 180-199 104 104 104 185 185 185 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 100 100 100 175 175 175 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 104 104 104 183 183 183 F F F 16 320-339 104 104 104 183 183 183 T T T 17 340-359 94 94 94 162 162 162 F F F 18 360-379 104 104 104 185 185 185 F F F 19 380-399 104 104 104 185 185 185 T T T 20 400-419 100 100 100 174 174 174 T T T 21 420-439 98 98 98 169 169 169 F F F 22 440-459 102 102 102 179 179 179 F F F 23 460-479 104 104 104 184 184 184 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 102 102 102 178 178 178 F F F 27 540-559 97 97 97 166 166 166 F F F 28 560-579 103 103 103 181 181 181 F F F 29 580-599 103 103 103 181 181 181 F F F Max gen: 104, Max flush: 185 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 157 157 157 F F F 1 020-039 102 102 102 178 178 178 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 102 102 102 178 178 178 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 104 104 104 185 185 185 F F F 6 120-139 104 104 104 185 185 185 F F F 7 140-159 104 104 104 183 183 183 F F F 8 160-179 103 103 103 181 181 181 F F F 9 180-199 104 104 104 185 185 185 F F F 10 200-219 104 104 104 185 185 185 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 100 100 100 175 175 175 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 104 104 104 183 183 183 F F F 16 320-339 105 105 105 186 186 186 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 104 104 104 185 185 185 F F F 19 380-399 105 105 105 186 186 186 F F F 20 400-419 105 105 105 186 186 186 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 102 102 102 179 179 179 F F F 23 460-479 104 104 104 184 184 184 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 102 102 102 178 178 178 F F F 27 540-559 97 97 97 166 166 166 F F F 28 560-579 103 103 103 181 181 181 F F F 29 580-599 103 103 103 181 181 181 F F F Max gen: 105, Max flush: 186 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.949768093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.994763733Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.995368806Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.995411286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.995426982Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.995442489Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608} {"msg":"Crucible d370656e-8639-453b-8063-3caf512f3a53 has session id: df1f66bf-521f-483a-80d6-17a477b6c000","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996173503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Upstairs opts: Upstairs UUID: d370656e-8639-453b-8063-3caf512f3a53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996218663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Crucible stats registered with UUID: d370656e-8639-453b-8063-3caf512f3a53","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996243308Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:51.996269991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996758021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996784955Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608} {"msg":"d370656e-8639-453b-8063-3caf512f3a53 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996855349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996874845Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"0","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996902148Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"0","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996926313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"1","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996953217Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"1","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.996979591Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"2","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997008694Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"2","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997035058Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997064092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997092865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"ds_connection connected from Ok(127.0.0.1:39493)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997122098Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"ds_connection connected from Ok(127.0.0.1:39544)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997149462Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"ds_connection connected from Ok(127.0.0.1:61447)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997185394Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997876158Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"1","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.99790906Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"0","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.997937434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"2","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.998533629Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"1","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.998562652Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"2","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.998633026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"0","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999093322Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999207036Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[0]R flush_numbers[0..12]: [157, 178, 170, 178, 169, 185, 185, 183, 181, 185, 185, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999270102Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[0]R generation[0..12]: [92, 102, 98, 102, 98, 104, 104, 104, 103, 104, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999289828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999304994Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[1]R flush_numbers[0..12]: [157, 178, 170, 178, 169, 185, 185, 183, 181, 185, 185, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999318931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[1]R generation[0..12]: [92, 102, 98, 102, 98, 104, 104, 104, 103, 104, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999332488Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999349984Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[2]R flush_numbers[0..12]: [157, 178, 170, 178, 169, 185, 185, 183, 181, 185, 185, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999363881Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[2]R generation[0..12]: [92, 102, 98, 102, 98, 104, 104, 104, 103, 104, 104, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999377248Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999390835Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999403552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999416499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999432585Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.99945466Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"":"downstairs","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999481604Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999494511Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"0","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999507688Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"1","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999520445Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"client":"2","":"downstairs","session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999533452Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.99954534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"d370656e-8639-453b-8063-3caf512f3a53 is now active with session: df1f66bf-521f-483a-80d6-17a477b6c000","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999557807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999571884Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608,"session_id":"df1f66bf-521f-483a-80d6-17a477b6c000"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:51.999584111Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 585 len 40960 data: 10 9 10 7 7 7 6 5 4 3 02/30 Read block 422 len 20480 03/30 Write block 27 len 40960 data: 7 8 7 7 7 8 8 7 7 7 04/30 Write block 198 len 24576 data: 7 8 8 7 10 7 05/30 Read block 5 len 12288 06/30 Read block 478 len 36864 07/30 Write block 177 len 8192 data: 11 10 08/30 Read block 292 len 28672 09/30 Write block 382 len 32768 data: 8 8 9 10 10 10 11 10 10/30 Flush 11/30 Write block 19 len 20480 data: 6 8 9 6 5 12/30 Write block 107 len 4096 data: 9 13/30 Read block 334 len 36864 14/30 Read block 121 len 12288 15/30 Read block 205 len 16384 16/30 Read block 311 len 16384 17/30 Read block 182 len 12288 18/30 Write block 461 len 32768 data: 5 6 7 8 8 10 10 11 19/30 Read block 35 len 28672 20/30 Read block 338 len 12288 21/30 Flush 22/30 Write block 529 len 36864 data: 10 9 7 6 7 10 10 11 8 23/30 Write block 338 len 8192 data: 7 8 24/30 Read block 555 len 36864 25/30 Read block 213 len 36864 26/30 Write block 543 len 24576 data: 9 10 10 9 9 7 27/30 Write block 62 len 20480 data: 6 7 8 9 9 28/30 Read block 231 len 24576 29/30 Read block 475 len 36864 30/30 Write block 108 len 36864 data: 12 9 7 7 7 8 8 11 13 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:52.020037939Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1608} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 188 188 188 F F F 1 020-039 106 106 106 188 188 188 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 102 102 102 178 178 178 T T T 4 080-099 98 98 98 169 169 169 F F F 5 100-119 106 106 106 188 188 188 T T T 6 120-139 104 104 104 185 185 185 F F F 7 140-159 104 104 104 183 183 183 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 106 106 106 187 187 187 F F F 10 200-219 106 106 106 187 187 187 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 100 100 100 175 175 175 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 104 104 104 183 183 183 F F F 16 320-339 105 105 105 186 186 186 T T T 17 340-359 94 94 94 162 162 162 F F F 18 360-379 104 104 104 185 185 185 F F F 19 380-399 106 106 106 187 187 187 F F F 20 400-419 105 105 105 186 186 186 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 102 102 102 179 179 179 F F F 23 460-479 106 106 106 188 188 188 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 102 102 102 178 178 178 T T T 27 540-559 97 97 97 166 166 166 T T T 28 560-579 103 103 103 181 181 181 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 106, Max flush: 188 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 188 188 188 F F F 1 020-039 106 106 106 188 188 188 F F F 2 040-059 98 98 98 170 170 170 F F F 3 060-079 107 107 107 189 189 189 F F F 4 080-099 98 98 98 169 169 169 F F F 5 100-119 107 107 107 189 189 189 F F F 6 120-139 104 104 104 185 185 185 F F F 7 140-159 104 104 104 183 183 183 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 106 106 106 187 187 187 F F F 10 200-219 106 106 106 187 187 187 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 100 100 100 175 175 175 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 104 104 104 183 183 183 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 104 104 104 185 185 185 F F F 19 380-399 106 106 106 187 187 187 F F F 20 400-419 105 105 105 186 186 186 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 102 102 102 179 179 179 F F F 23 460-479 106 106 106 188 188 188 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 107 107 107 189 189 189 F F F 27 540-559 107 107 107 189 189 189 F F F 28 560-579 103 103 103 181 181 181 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 107, Max flush: 189 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.26559156Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:56.309333123Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.309747409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.342275542Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.342969895Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.343028922Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.343044378Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.343058815Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616} {"msg":"Crucible 2187badb-d7c3-413b-a8b9-dfb328f75b9b has session id: a1b3d68d-f0c5-44d1-b56a-413f736cb092","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.343729853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Upstairs opts: Upstairs UUID: 2187badb-d7c3-413b-a8b9-dfb328f75b9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.34378502Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Crucible stats registered with UUID: 2187badb-d7c3-413b-a8b9-dfb328f75b9b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.343801947Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:56.343815293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.34431522Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344348613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616} {"msg":"2187badb-d7c3-413b-a8b9-dfb328f75b9b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344403131Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344430414Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"0","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344454789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"0","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344478393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"1","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344505287Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"1","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.3445376Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"2","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344570193Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"2","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344600496Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344629539Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344654014Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"ds_connection connected from Ok(127.0.0.1:64805)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344678438Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"ds_connection connected from Ok(127.0.0.1:58001)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344704232Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"ds_connection connected from Ok(127.0.0.1:60248)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.344733086Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.345440925Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"1","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.345469739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"2","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.34550935Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"0","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346108754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"1","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346137258Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"0","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346162562Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"2","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346664648Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346696141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[0]R flush_numbers[0..12]: [188, 188, 170, 189, 169, 189, 185, 183, 187, 187, 187, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346779122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[0]R generation[0..12]: [106, 106, 98, 107, 98, 107, 104, 104, 106, 106, 106, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346798658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346813035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[1]R flush_numbers[0..12]: [188, 188, 170, 189, 169, 189, 185, 183, 187, 187, 187, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346826642Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[1]R generation[0..12]: [106, 106, 98, 107, 98, 107, 104, 104, 106, 106, 106, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346840408Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346854645Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[2]R flush_numbers[0..12]: [188, 188, 170, 189, 169, 189, 185, 183, 187, 187, 187, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346868682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[2]R generation[0..12]: [106, 106, 98, 107, 98, 107, 104, 104, 106, 106, 106, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346882299Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346896046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346908773Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.34692188Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346934587Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346950734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346970459Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346982286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"0","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.346995273Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"1","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.347017648Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"client":"2","":"downstairs","session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.347031365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.347043502Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"2187badb-d7c3-413b-a8b9-dfb328f75b9b is now active with session: a1b3d68d-f0c5-44d1-b56a-413f736cb092","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.34705601Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.347067707Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616,"session_id":"a1b3d68d-f0c5-44d1-b56a-413f736cb092"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:22:56.347079634Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 364 len 20480 data: 10 10 6 6 7 02/30 Read block 223 len 36864 03/30 Read block 239 len 20480 04/30 Read block 315 len 4096 05/30 Read block 319 len 28672 06/30 Read block 220 len 40960 07/30 Write block 195 len 40960 data: 9 8 7 8 9 9 8 11 8 6 08/30 Read block 81 len 28672 09/30 Read block 510 len 8192 10/30 Read block 496 len 40960 11/30 Read block 165 len 40960 12/30 Read block 117 len 20480 13/30 Flush 14/30 Write block 99 len 16384 data: 10 11 12 14 15/30 Read block 232 len 28672 16/30 Read block 268 len 16384 17/30 Read block 550 len 16384 18/30 Read block 475 len 24576 19/30 Read block 218 len 16384 20/30 Write block 37 len 40960 data: 8 11 9 10 10 9 13 13 14 10 21/30 Read block 166 len 24576 22/30 Read block 339 len 40960 23/30 Write block 388 len 12288 data: 12 11 11 24/30 Read block 241 len 32768 25/30 Write block 10 len 12288 data: 7 6 8 26/30 Write block 244 len 12288 data: 8 8 9 27/30 Read block 120 len 36864 28/30 Write block 361 len 36864 data: 10 7 8 11 11 7 7 8 5 29/30 Read block 31 len 40960 30/30 Read block 413 len 36864 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:22:56.368873618Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1616} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 188 188 188 T T T 1 020-039 106 106 106 188 188 188 T T T 2 040-059 98 98 98 170 170 170 T T T 3 060-079 107 107 107 189 189 189 F F F 4 080-099 98 98 98 169 169 169 T T T 5 100-119 107 107 107 189 189 189 T T T 6 120-139 104 104 104 185 185 185 F F F 7 140-159 104 104 104 183 183 183 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 108 108 108 190 190 190 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 100 100 100 175 175 175 T T T 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 104 104 104 183 183 183 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 108 108 108 190 190 190 T T T 19 380-399 106 106 106 187 187 187 T T T 20 400-419 105 105 105 186 186 186 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 102 102 102 179 179 179 F F F 23 460-479 106 106 106 188 188 188 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 107 107 107 189 189 189 F F F 27 540-559 107 107 107 189 189 189 F F F 28 560-579 103 103 103 181 181 181 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 108, Max flush: 190 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 109 109 109 191 191 191 F F F 2 040-059 109 109 109 191 191 191 F F F 3 060-079 107 107 107 189 189 189 F F F 4 080-099 109 109 109 191 191 191 F F F 5 100-119 109 109 109 191 191 191 F F F 6 120-139 104 104 104 185 185 185 F F F 7 140-159 104 104 104 183 183 183 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 108 108 108 190 190 190 F F F 10 200-219 108 108 108 190 190 190 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 109 109 109 191 191 191 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 104 104 104 183 183 183 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 94 94 94 162 162 162 F F F 18 360-379 109 109 109 191 191 191 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 105 105 105 186 186 186 F F F 21 420-439 98 98 98 169 169 169 F F F 22 440-459 102 102 102 179 179 179 F F F 23 460-479 106 106 106 188 188 188 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 107 107 107 189 189 189 F F F 27 540-559 107 107 107 189 189 189 F F F 28 560-579 103 103 103 181 181 181 F F F 29 580-599 106 106 106 187 187 187 F F F Max gen: 109, Max flush: 191 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.657262939Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.702224446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.702981235Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.703037082Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.703058767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.703081362Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624} {"msg":"Crucible b3cf6f90-7a2d-4a6b-8660-4b4f5fa03b86 has session id: 4bacc7f5-f3ce-4dc3-b859-bee167eab80f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.703786732Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Upstairs opts: Upstairs UUID: b3cf6f90-7a2d-4a6b-8660-4b4f5fa03b86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.703831762Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Crucible stats registered with UUID: b3cf6f90-7a2d-4a6b-8660-4b4f5fa03b86","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.703855387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:00.703876302Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704367201Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704393865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624} {"msg":"b3cf6f90-7a2d-4a6b-8660-4b4f5fa03b86 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704469848Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704488853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"0","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704511848Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"0","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704525665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"1","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704539682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"1","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704552649Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"2","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704566286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"2","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704625942Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704645558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704659605Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53251)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704743556Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"ds_connection connected from Ok(127.0.0.1:52228)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.704778128Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38668)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.70494353Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.705753067Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"1","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.705798857Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"0","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.706550866Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"1","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.706591817Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"0","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.70662329Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"2","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707059611Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707158319Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707179274Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"2","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[0]R flush_numbers[0..12]: [191, 191, 191, 189, 191, 191, 185, 183, 187, 190, 190, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707717912Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[0]R generation[0..12]: [109, 109, 109, 107, 109, 109, 104, 104, 106, 108, 108, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707742157Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707757433Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[1]R flush_numbers[0..12]: [191, 191, 191, 189, 191, 191, 185, 183, 187, 190, 190, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.70777168Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[1]R generation[0..12]: [109, 109, 109, 107, 109, 109, 104, 104, 106, 108, 108, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707785647Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707799644Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[2]R flush_numbers[0..12]: [191, 191, 191, 189, 191, 191, 185, 183, 187, 190, 190, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707817709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[2]R generation[0..12]: [109, 109, 109, 107, 109, 109, 104, 104, 106, 108, 108, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707831776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707845673Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.70785849Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"Wait for a query_work_queue command to finish before sending IO } {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707878076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707890863Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.70790372Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"":"downstairs","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707916797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707928614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"0","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707942001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"1","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707955088Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"client":"2","":"downstairs","session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707968285Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.707980332Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"b3cf6f90-7a2d-4a6b-8660-4b4f5fa03b86 is now active with session: 4bacc7f5-f3ce-4dc3-b859-bee167eab80f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.70799299Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.708004797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624,"session_id":"4bacc7f5-f3ce-4dc3-b859-bee167eab80f"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:00.708019524Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 563 len 32768 data: 8 6 3 4 5 7 9 9 02/30 Write block 559 len 12288 data: 4 6 5 03/30 Read block 219 len 16384 04/30 Write block 201 len 24576 data: 9 12 9 7 7 7 05/30 Read block 92 len 8192 06/30 Write block 69 len 4096 data: 13 07/30 Write block 63 len 24576 data: 8 9 10 10 10 11 08/30 Write block 50 len 28672 data: 6 6 6 7 8 7 7 09/30 Write block 362 len 36864 data: 8 9 12 12 8 8 9 6 7 10/30 Flush 11/30 Flush 12/30 Write block 532 len 12288 data: 7 8 11 13/30 Write block 566 len 4096 data: 5 14/30 Write block 309 len 32768 data: 6 5 4 4 4 6 7 8 15/30 Read block 490 len 28672 16/30 Read block 554 len 20480 17/30 Write block 589 len 24576 data: 8 8 7 6 5 4 18/30 Write block 244 len 32768 data: 9 9 10 7 9 9 7 8 19/30 Read block 288 len 24576 20/30 Write block 549 len 12288 data: 7 7 7 21/30 Write block 435 len 32768 data: 9 8 9 11 10 9 10 11 22/30 Write block 345 len 40960 data: 5 5 7 9 10 8 8 9 12 12 23/30 Read block 186 len 20480 24/30 Read block 157 len 20480 25/30 Read block 570 len 24576 26/30 Write block 76 len 8192 data: 4 5 27/30 Read block 62 len 12288 28/30 Write block 576 len 32768 data: 11 11 10 12 10 11 12 12 29/30 Read block 432 len 36864 30/30 Read block 88 len 8192 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 3 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:00.727379569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1624} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 109 109 109 191 191 191 F F F 2 040-059 110 110 110 192 192 192 F F F 3 060-079 110 110 110 192 192 192 T T T 4 080-099 109 109 109 191 191 191 F F F 5 100-119 109 109 109 191 191 191 F F F 6 120-139 104 104 104 185 185 185 F F F 7 140-159 104 104 104 183 183 183 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 108 108 108 190 190 190 F F F 10 200-219 110 110 110 192 192 192 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 109 109 109 191 191 191 T T T 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 104 104 104 183 183 183 T T T 16 320-339 107 107 107 189 189 189 F F F 17 340-359 94 94 94 162 162 162 T T T 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 105 105 105 186 186 186 F F F 21 420-439 98 98 98 169 169 169 T T T 22 440-459 102 102 102 179 179 179 T T T 23 460-479 106 106 106 188 188 188 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 107 107 107 189 189 189 T T T 27 540-559 110 110 110 192 192 192 T T T 28 560-579 110 110 110 192 192 192 T T T 29 580-599 106 106 106 187 187 187 T T T Max gen: 110, Max flush: 192 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 109 109 109 191 191 191 F F F 2 040-059 110 110 110 192 192 192 F F F 3 060-079 111 111 111 193 193 193 F F F 4 080-099 109 109 109 191 191 191 F F F 5 100-119 109 109 109 191 191 191 F F F 6 120-139 104 104 104 185 185 185 F F F 7 140-159 104 104 104 183 183 183 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 108 108 108 190 190 190 F F F 10 200-219 110 110 110 192 192 192 F F F 11 220-239 103 103 103 181 181 181 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 98 98 98 169 169 169 F F F 14 280-299 98 98 98 171 171 171 F F F 15 300-319 111 111 111 193 193 193 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 111 111 111 193 193 193 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 105 105 105 186 186 186 F F F 21 420-439 111 111 111 193 193 193 F F F 22 440-459 111 111 111 193 193 193 F F F 23 460-479 106 106 106 188 188 188 F F F 24 480-499 104 104 104 182 182 182 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 111 111 111 193 193 193 F F F 27 540-559 111 111 111 193 193 193 F F F 28 560-579 111 111 111 193 193 193 F F F 29 580-599 111 111 111 193 193 193 F F F Max gen: 111, Max flush: 193 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:04.995592779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.04301003Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.046075406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.046121675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.04614384Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.046158997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632} {"msg":"Crucible b9015290-69b8-43cb-83c2-ee9658c3a26f has session id: 24d1d097-5e7d-4148-b514-a0887ad6ca6c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.046727918Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Upstairs opts: Upstairs UUID: b9015290-69b8-43cb-83c2-ee9658c3a26f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.046815618Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Crucible stats registered with UUID: b9015290-69b8-43cb-83c2-ee9658c3a26f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.046861168Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:05.046881093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047303108Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047330091Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632} {"msg":"b9015290-69b8-43cb-83c2-ee9658c3a26f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047412913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047430999Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"0","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047454313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"0","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.04746824Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"1","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047482747Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"1","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047503242Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"2","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047530176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"2","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.04755643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047584874Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047611628Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52911)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047638102Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39222)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047665605Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55588)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.047705656Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.048469293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"2","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.048497347Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"0","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.048708989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"1","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049137872Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"0","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049177013Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"2","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049404222Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"1","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049858359Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049883923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[0]R flush_numbers[0..12]: [191, 191, 192, 193, 191, 191, 185, 183, 187, 190, 192, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049902279Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[0]R generation[0..12]: [109, 109, 110, 111, 109, 109, 104, 104, 106, 108, 110, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049916825Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049931092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[1]R flush_numbers[0..12]: [191, 191, 192, 193, 191, 191, 185, 183, 187, 190, 192, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049945349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[1]R generation[0..12]: [109, 109, 110, 111, 109, 109, 104, 104, 106, 108, 110, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049959326Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.049973373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[2]R flush_numbers[0..12]: [191, 191, 192, 193, 191, 191, 185, 183, 187, 190, 192, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.04998684Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[2]R generation[0..12]: [109, 109, 110, 111, 109, 109, 104, 104, 106, 108, 110, 103]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050000666Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050014633Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050027251Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050040268Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050052955Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050065642Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"":"downstairs","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050082818Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050095225Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"0","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:23:05.050108642Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"1","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050127708Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"client":"2","":"downstairs","session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050140905Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050152902Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"b9015290-69b8-43cb-83c2-ee9658c3a26f is now active with session: 24d1d097-5e7d-4148-b514-a0887ad6ca6c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050165979Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050177706Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632,"session_id":"24d1d097-5e7d-4148-b514-a0887ad6ca6c"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:05.050189614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 79 len 4096 data: 6 02/30 Write block 348 len 16384 data: 10 11 9 9 03/30 Write block 311 len 28672 data: 5 5 5 7 8 9 9 04/30 Write block 194 len 4096 data: 11 05/30 Write block 275 len 40960 data: 5 4 5 8 7 8 10 9 6 6 06/30 Flush 07/30 Write block 146 len 32768 data: 14 12 11 10 8 6 3 5 08/30 Write block 306 len 28672 data: 5 6 6 7 6 6 6 09/30 Flush 10/30 Write block 443 len 36864 data: 11 12 11 8 10 9 10 12 11 11/30 Read block 322 len 8192 12/30 Write block 144 len 36864 data: 10 12 15 13 12 11 9 7 4 13/30 Write block 139 len 24576 data: 7 8 7 9 9 11 14/30 Write block 482 len 32768 data: 6 6 8 8 11 13 13 11 15/30 Read block 211 len 8192 16/30 Write block 412 len 40960 data: 8 8 8 9 8 7 6 6 8 6 17/30 Read block 571 len 24576 18/30 Read block 250 len 4096 19/30 Write block 580 len 8192 data: 11 12 20/30 Write block 222 len 16384 data: 12 17 15 17 21/30 Read block 218 len 28672 22/30 Write block 477 len 4096 data: 7 23/30 Write block 426 len 40960 data: 8 8 8 9 7 7 10 10 11 10 24/30 Write block 595 len 12288 data: 2 2 2 25/30 Write block 58 len 24576 data: 6 6 6 8 7 9 26/30 Read block 484 len 36864 27/30 Read block 66 len 40960 28/30 Write block 101 len 28672 data: 13 15 11 7 8 8 10 29/30 Write block 95 len 16384 data: 12 12 13 12 30/30 Write block 522 len 20480 data: 16 14 14 13 13 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 9 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 7 Sent Done Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 3 8 Done 18 19 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:05.067966548Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1632} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 109 109 109 191 191 191 F F F 2 040-059 110 110 110 192 192 192 T T T 3 060-079 112 112 112 194 194 194 T T T 4 080-099 109 109 109 191 191 191 T T T 5 100-119 109 109 109 191 191 191 T T T 6 120-139 104 104 104 185 185 185 T T T 7 140-159 112 112 112 195 195 195 T T T 8 160-179 106 106 106 187 187 187 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 110 110 110 192 192 192 F F F 11 220-239 103 103 103 181 181 181 T T T 12 240-259 111 111 111 193 193 193 F F F 13 260-279 112 112 112 194 194 194 F F F 14 280-299 112 112 112 194 194 194 F F F 15 300-319 112 112 112 195 195 195 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 105 105 105 186 186 186 T T T 21 420-439 111 111 111 193 193 193 T T T 22 440-459 111 111 111 193 193 193 T T T 23 460-479 106 106 106 188 188 188 T T T 24 480-499 104 104 104 182 182 182 T T T 25 500-519 104 104 104 182 182 182 F F F 26 520-539 111 111 111 193 193 193 T T T 27 540-559 111 111 111 193 193 193 F F F 28 560-579 111 111 111 193 193 193 F F F 29 580-599 111 111 111 193 193 193 T T T Max gen: 112, Max flush: 195 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 109 109 109 191 191 191 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 113 113 113 196 196 196 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 113 113 113 196 196 196 F F F 8 160-179 106 106 106 187 187 187 F F F 9 180-199 112 112 112 194 194 194 F F F 10 200-219 110 110 110 192 192 192 F F F 11 220-239 113 113 113 196 196 196 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 112 112 112 194 194 194 F F F 14 280-299 112 112 112 194 194 194 F F F 15 300-319 112 112 112 195 195 195 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 113 113 113 196 196 196 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 113 113 113 196 196 196 F F F 23 460-479 113 113 113 196 196 196 F F F 24 480-499 113 113 113 196 196 196 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 113 113 113 196 196 196 F F F 27 540-559 111 111 111 193 193 193 F F F 28 560-579 111 111 111 193 193 193 F F F 29 580-599 113 113 113 196 196 196 F F F Max gen: 113, Max flush: 196 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.354906554Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:19.398285289Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.398660804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.430966378Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.431595075Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.431630787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.431644814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.431658981Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640} {"msg":"Crucible c31c2cc6-08ba-4821-92c5-0861d54cf02d has session id: 5bd8b93a-f29c-46a0-a1bf-5d287b5603af","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.43232515Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Upstairs opts: Upstairs UUID: c31c2cc6-08ba-4821-92c5-0861d54cf02d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432386556Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Crucible stats registered with UUID: c31c2cc6-08ba-4821-92c5-0861d54cf02d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432410441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:19.432426057Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432814049Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432841473Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640} {"msg":"c31c2cc6-08ba-4821-92c5-0861d54cf02d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432883034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.4328989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"0","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432921775Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"0","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432935552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"1","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432949579Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"1","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432962586Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"2","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432976212Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"2","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.432998078Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.433022422Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.433045777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"ds_connection connected from Ok(127.0.0.1:55205)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.433174438Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"ds_connection connected from Ok(127.0.0.1:38235)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.433230485Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"ds_connection connected from Ok(127.0.0.1:43258)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.433330542Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.43404513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"1","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.434085701Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"2","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.434116974Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"0","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.434727146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"2","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.43475511Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"0","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.434774345Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"1","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435323101Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435346516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[0]R flush_numbers[0..12]: [191, 191, 196, 196, 196, 196, 196, 196, 187, 194, 192, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435359953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[0]R generation[0..12]: [109, 109, 113, 113, 113, 113, 113, 113, 106, 112, 110, 113]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435374369Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435388856Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[1]R flush_numbers[0..12]: [191, 191, 196, 196, 196, 196, 196, 196, 187, 194, 192, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435406282Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[1]R generation[0..12]: [109, 109, 113, 113, 113, 113, 113, 113, 106, 112, 110, 113]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435420149Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435434096Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[2]R flush_numbers[0..12]: [191, 191, 196, 196, 196, 196, 196, 196, 187, 194, 192, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435447713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[2]R generation[0..12]: [109, 109, 113, 113, 113, 113, 113, 113, 106, 112, 110, 113]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.43546116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435474957Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435487754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435500631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435513308Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435526195Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"":"downstairs","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435539062Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435550809Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"0","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435563877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"1Wait for a query_work_queue command to finish before sending IO ","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435584892Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"client":"2","":"downstairs","session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435600138Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435612326Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"c31c2cc6-08ba-4821-92c5-0861d54cf02d is now active with session: 5bd8b93a-f29c-46a0-a1bf-5d287b5603af","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435624763Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.43563646Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640,"session_id":"5bd8b93a-f29c-46a0-a1bf-5d287b5603af"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:19.435648507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 30 len 8192 data: 8 8 02/30 Write block 246 len 28672 data: 11 8 10 10 8 9 6 03/30 Write block 230 len 16384 data: 15 14 11 10 04/30 Write block 263 len 24576 data: 11 12 12 13 12 13 05/30 Read block 242 len 16384 06/30 Read block 90 len 40960 07/30 Read block 300 len 36864 08/30 Read block 173 len 20480 09/30 Write block 215 len 40960 data: 8 9 9 10 9 8 9 13 18 16 10/30 Read block 509 len 32768 11/30 Read block 93 len 24576 12/30 Read block 387 len 8192 13/30 Read block 255 len 8192 14/30 Write block 291 len 20480 data: 4 4 3 3 2 15/30 Write block 144 len 24576 data: 12 13 16 14 13 12 16/30 Write block 159 len 36864 data: 11 11 10 11 11 9 9 9 9 17/30 Write block 181 len 16384 data: 12 11 11 9 18/30 Read block 162 len 12288 19/30 Write block 61 len 24576 data: 9 8 10 10 11 11 20/30 Read block 524 len 28672 21/30 Read block 534 len 8192 22/30 Write block 543 len 28672 data: 10 11 11 10 10 8 8 23/30 Write block 481 len 8192 data: 8 7 24/30 Read block 159 len 12288 25/30 Write block 579 len 28672 data: 13 12 13 13 13 11 11 26/30 Read block 320 len 16384 27/30 Read block 529 len 8192 28/30 Write block 253 len 4096 data: 6 29/30 Write block 231 len 28672 data: 15 12 11 11 11 11 11 30/30 Read block 57 len 40960 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:19.455239001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1640} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 109 109 109 191 191 191 T T T 2 040-059 113 113 113 196 196 196 F F F 3 060-079 113 113 113 196 196 196 T T T 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 113 113 113 196 196 196 T T T 8 160-179 106 106 106 187 187 187 T T T 9 180-199 112 112 112 194 194 194 T T T 10 200-219 110 110 110 192 192 192 T T T 11 220-239 113 113 113 196 196 196 T T T 12 240-259 111 111 111 193 193 193 T T T 13 260-279 112 112 112 194 194 194 T T T 14 280-299 112 112 112 194 194 194 T T T 15 300-319 112 112 112 195 195 195 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 113 113 113 196 196 196 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 113 113 113 196 196 196 F F F 23 460-479 113 113 113 196 196 196 F F F 24 480-499 113 113 113 196 196 196 T T T 25 500-519 104 104 104 182 182 182 F F F 26 520-539 113 113 113 196 196 196 F F F 27 540-559 111 111 111 193 193 193 T T T 28 560-579 111 111 111 193 193 193 T T T 29 580-599 113 113 113 196 196 196 T T T Max gen: 113, Max flush: 196 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 114 114 114 197 197 197 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 197 197 197 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 114 114 114 197 197 197 F F F 8 160-179 114 114 114 197 197 197 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 114 114 114 197 197 197 F F F 11 220-239 114 114 114 197 197 197 F F F 12 240-259 114 114 114 197 197 197 F F F 13 260-279 114 114 114 197 197 197 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 112 112 112 195 195 195 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 113 113 113 196 196 196 F F F 21 420-439 113 113 113 196 196 196 F F F 22 440-459 113 113 113 196 196 196 F F F 23 460-479 113 113 113 196 196 196 F F F 24 480-499 114 114 114 197 197 197 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 113 113 113 196 196 196 F F F 27 540-559 114 114 114 197 197 197 F F F 28 560-579 114 114 114 197 197 197 F F F 29 580-599 114 114 114 197 197 197 F F F Max gen: 114, Max flush: 197 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.778493765Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.825276379Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.826132535Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.826175675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.826191192Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.826205609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648} {"msg":"Crucible 85f5a3c5-a6c1-4d5c-a5d2-a160c9528826 has session id: aa3bb33f-113f-4c69-b717-fa6e02da910c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.826727071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Upstairs opts: Upstairs UUID: 85f5a3c5-a6c1-4d5c-a5d2-a160c9528826, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.826771201Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Crucible stats registered with UUID: 85f5a3c5-a6c1-4d5c-a5d2-a160c9528826","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.826795005Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:23.826815711Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827280565Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827311898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648} {"msg":"85f5a3c5-a6c1-4d5c-a5d2-a160c9528826 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827374674Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.82739375Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"0","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827417574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"0","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827431811Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"1","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827445788Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"1","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827458455Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"2","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827472122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"2","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827484899Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827498316Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827618849Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52998)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827653971Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59321)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827680755Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"ds_connection connected from Ok(127.0.0.1:35005)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.827704359Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.828329718Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"0","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.82836361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"1","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.828618113Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"2","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829150932Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"1","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829182945Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"0","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829212198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"2","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829755595Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829781399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[0]R flush_numbers[0..12]: [191, 197, 196, 197, 196, 196, 196, 197, 197, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829806573Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[0]R generation[0..12]: [109, 114, 113, 114, 113, 113, 113, 114, 114, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.82982162Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829835877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[1]R flush_numbers[0..12]: [191, 197, 196, 197, 196, 196, 196, 197, 197, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829849923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[1]R generation[0..12]: [109, 114, 113, 114, 113, 113, 113, 114, 114, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.82986363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829877797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[2]R flush_numbers[0..12]: [191, 197, 196, 197, 196, 196, 196, 197, 197, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829891214Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[2]R generation[0..12]: [109, 114, 113, 114, 113, 113, 113, 114, 114, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.82990841Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829922547Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Max found gen is 115","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829935184Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Generation requested: 116 >= found:115","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829948111Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.829960848Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:23:23.829973506Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"":"downstairs","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.830055947Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.830077152Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"0","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.830091169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"1","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.830104776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"client":"2","":"downstairs","session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.830117923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.83012997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"85f5a3c5-a6c1-4d5c-a5d2-a160c9528826 is now active with session: aa3bb33f-113f-4c69-b717-fa6e02da910c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.830142607Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.830154445Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648,"session_id":"aa3bb33f-113f-4c69-b717-fa6e02da910c"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:23.830166402Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 480 len 16384 data: 7 9 8 7 02/30 Read block 243 len 16384 03/30 Read block 429 len 24576 04/30 Write block 239 len 20480 data: 8 9 12 12 11 05/30 Read block 383 len 4096 06/30 Read block 492 len 24576 07/30 Read block 223 len 32768 08/30 Read block 569 len 4096 09/30 Flush 10/30 Write block 287 len 28672 data: 7 4 6 6 5 5 4 11/30 Read block 134 len 4096 12/30 Write block 278 len 28672 data: 9 8 9 11 10 7 7 13/30 Write block 567 len 28672 data: 6 8 10 10 10 11 10 14/30 Read block 438 len 20480 15/30 Read block 291 len 16384 16/30 Write block 431 len 40960 data: 8 11 11 12 11 9 10 12 11 10 17/30 Read block 456 len 40960 18/30 Write block 424 len 16384 data: 7 8 9 9 19/30 Read block 538 len 40960 20/30 Read block 574 len 16384 21/30 Write block 521 len 24576 data: 14 17 15 15 14 14 22/30 Read block 574 len 8192 23/30 Read block 306 len 24576 24/30 Read block 133 len 32768 25/30 Flush 26/30 Write block 25 len 40960 data: 5 8 8 9 8 9 9 9 9 8 27/30 Read block 416 len 24576 28/30 Write block 489 len 12288 data: 12 9 7 29/30 Read block 138 len 16384 30/30 Read block 139 len 4096 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:23.851205737Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1648} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 114 114 114 197 197 197 T T T 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 197 197 197 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 114 114 114 197 197 197 F F F 8 160-179 114 114 114 197 197 197 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 114 114 114 197 197 197 F F F 11 220-239 116 116 116 198 198 198 F F F 12 240-259 116 116 116 198 198 198 F F F 13 260-279 116 116 116 199 199 199 F F F 14 280-299 116 116 116 199 199 199 F F F 15 300-319 112 112 112 195 195 195 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 113 113 113 196 196 196 F F F 21 420-439 116 116 116 199 199 199 F F F 22 440-459 116 116 116 199 199 199 F F F 23 460-479 113 113 113 196 196 196 F F F 24 480-499 116 116 116 198 198 198 T T T 25 500-519 104 104 104 182 182 182 F F F 26 520-539 116 116 116 199 199 199 F F F 27 540-559 114 114 114 197 197 197 F F F 28 560-579 116 116 116 199 199 199 F F F 29 580-599 114 114 114 197 197 197 F F F Max gen: 116, Max flush: 199 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 117 117 117 200 200 200 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 197 197 197 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 113 113 113 196 196 196 F F F 7 140-159 114 114 114 197 197 197 F F F 8 160-179 114 114 114 197 197 197 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 114 114 114 197 197 197 F F F 11 220-239 116 116 116 198 198 198 F F F 12 240-259 116 116 116 198 198 198 F F F 13 260-279 116 116 116 199 199 199 F F F 14 280-299 116 116 116 199 199 199 F F F 15 300-319 112 112 112 195 195 195 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 113 113 113 196 196 196 F F F 21 420-439 116 116 116 199 199 199 F F F 22 440-459 116 116 116 199 199 199 F F F 23 460-479 113 113 113 196 196 196 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 116 116 116 199 199 199 F F F 27 540-559 114 114 114 197 197 197 F F F 28 560-579 116 116 116 199 199 199 F F F 29 580-599 114 114 114 197 197 197 F F F Max gen: 117, Max flush: 200 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.075477239Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.121077041Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.121765015Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.121807886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.121823122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.121837499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656} {"msg":"Crucible f7891dc3-6f8a-4f8f-b666-b0dcd9f18d14 has session id: 1f33bb82-a41c-4047-8904-a6d5ca1badda","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.122466306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Upstairs opts: Upstairs UUID: f7891dc3-6f8a-4f8f-b666-b0dcd9f18d14, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.122511366Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Crucible stats registered with UUID: f7891dc3-6f8a-4f8f-b666-b0dcd9f18d14","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.12253979Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:28.122559785Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123088796Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123129146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656} {"msg":"f7891dc3-6f8a-4f8f-b666-b0dcd9f18d14 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.12319936Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.12324313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"0","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123286891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"0","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123326192Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"1","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123356785Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"1","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123384039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"2","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123407283Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"2","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.12342173Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123435457Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123454813Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"ds_connection connected from Ok(127.0.0.1:39595)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123483396Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"ds_connection connected from Ok(127.0.0.1:65321)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123513049Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"ds_connection connected from Ok(127.0.0.1:52401)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.123652288Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.124336283Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"2","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.124374934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"1","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.124402828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"0","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.124885629Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"0","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125023727Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"1","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125159396Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"2","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125520475Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125607045Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[0]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 196, 197, 197, 197, 197, 198]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.12571565Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[0]R generation[0..12]: [109, 117, 113, 114, 113, 113, 113, 114, 114, 114, 114, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125736476Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125751013Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[1]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 196, 197, 197, 197, 197, 198]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125769738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[1]R generation[0..12]: [109, 117, 113, 114, 113, 113, 113, 114, 114, 114, 114, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125784005Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125797842Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[2]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 196, 197, 197, 197, 197, 198]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125811669Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[2]R generation[0..12]: [109, 117, 113, 114, 113, 113, 113, 114, 114, 114, 114, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125825156Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125838833Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.12585166Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125864617Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125877574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125890401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"":"downstairs","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125903798Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656Wait for a query_work_queue command to finish before sending IO ,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125928812Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"0","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125943779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"1","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125957286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"client":"2","":"downstairs","session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125970803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.1259856Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"f7891dc3-6f8a-4f8f-b666-b0dcd9f18d14 is now active with session: 1f33bb82-a41c-4047-8904-a6d5ca1badda","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.125998626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.126010554Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656,"session_id":"1f33bb82-a41c-4047-8904-a6d5ca1badda"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:28.126022691Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 446 len 16384 data: 9 11 10 11 02/30 Write block 548 len 40960 data: 9 9 8 8 4 3 3 4 4 4 03/30 Read block 318 len 32768 04/30 Read block 192 len 16384 05/30 Write block 402 len 8192 data: 15 15 06/30 Read block 487 len 4096 07/30 Read block 226 len 36864 08/30 Read block 241 len 28672 09/30 Write block 127 len 12288 data: 6 6 6 10/30 Read block 549 len 28672 11/30 Read block 533 len 32768 12/30 Read block 124 len 16384 13/30 Write block 577 len 20480 data: 12 11 14 13 14 14/30 Read block 134 len 4096 15/30 Write block 280 len 16384 data: 10 12 11 8 16/30 Read block 131 len 20480 17/30 Write block 422 len 8192 data: 5 7 18/30 Write block 568 len 20480 data: 9 11 11 11 12 19/30 Write block 422 len 16384 data: 6 8 8 9 20/30 Flush 21/30 Write block 165 len 20480 data: 10 10 10 8 10 22/30 Flush 23/30 Flush 24/30 Write block 577 len 40960 data: 13 12 15 14 15 14 14 12 12 10 25/30 Read block 318 len 4096 26/30 Read block 204 len 28672 27/30 Write block 232 len 16384 data: 13 12 12 12 28/30 Read block 563 len 16384 29/30 Read block 104 len 32768 30/30 Read block 476 len 4096 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:28.145335037Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1656} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 117 117 117 200 200 200 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 197 197 197 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 118 118 118 201 201 201 F F F 7 140-159 114 114 114 197 197 197 F F F 8 160-179 118 118 118 202 202 202 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 114 114 114 197 197 197 F F F 11 220-239 116 116 116 198 198 198 T T T 12 240-259 116 116 116 198 198 198 F F F 13 260-279 116 116 116 199 199 199 F F F 14 280-299 118 118 118 201 201 201 F F F 15 300-319 112 112 112 195 195 195 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 118 118 118 201 201 201 F F F 21 420-439 118 118 118 201 201 201 F F F 22 440-459 118 118 118 201 201 201 F F F 23 460-479 113 113 113 196 196 196 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 116 116 116 199 199 199 F F F 27 540-559 118 118 118 201 201 201 F F F 28 560-579 118 118 118 201 201 201 T T T 29 580-599 118 118 118 201 201 201 T T T Max gen: 118, Max flush: 202 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 117 117 117 200 200 200 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 197 197 197 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 118 118 118 201 201 201 F F F 7 140-159 114 114 114 197 197 197 F F F 8 160-179 118 118 118 202 202 202 F F F 9 180-199 114 114 114 197 197 197 F F F 10 200-219 114 114 114 197 197 197 F F F 11 220-239 119 119 119 203 203 203 F F F 12 240-259 116 116 116 198 198 198 F F F 13 260-279 116 116 116 199 199 199 F F F 14 280-299 118 118 118 201 201 201 F F F 15 300-319 112 112 112 195 195 195 F F F 16 320-339 107 107 107 189 189 189 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 F F F 20 400-419 118 118 118 201 201 201 F F F 21 420-439 118 118 118 201 201 201 F F F 22 440-459 118 118 118 201 201 201 F F F 23 460-479 113 113 113 196 196 196 F F F 24 480-499 117 117 117 200 200 200 F F F 25 500-519 104 104 104 182 182 182 F F F 26 520-539 116 116 116 199 199 199 F F F 27 540-559 118 118 118 201 201 201 F F F 28 560-579 119 119 119 203 203 203 F F F 29 580-599 119 119 119 203 203 203 F F F Max gen: 119, Max flush: 203 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.37353117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.418731493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.419420827Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.419474015Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.4194981Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.419516825Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664} {"msg":"Crucible 707bec91-510a-4b58-97a5-0c5909085af7 has session id: a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.420284142Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Upstairs opts: Upstairs UUID: 707bec91-510a-4b58-97a5-0c5909085af7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.420327942Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Crucible stats registered with UUID: 707bec91-510a-4b58-97a5-0c5909085af7","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.420352006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:32.420372092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.420879827Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.42091224Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664} {"msg":"707bec91-510a-4b58-97a5-0c5909085af7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.420942373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.420956899Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"0","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.420979744Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"0","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.420993391Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"1","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421007388Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"1","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421032632Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"2","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421058256Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"2","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421082581Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421105196Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421127211Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45450)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421243954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43561)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421427383Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63541)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.421498816Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.422083264Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"1","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.422123205Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"0","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.422318511Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"2","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.422791804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"0","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.422820837Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"1","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.422878564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"2","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423369533Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423400336Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[0]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 201, 197, 202, 197, 197, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423430259Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[0]R generation[0..12]: [109, 117, 113, 114, 113, 113, 118, 114, 118, 114, 114, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423445826Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423460292Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[1]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 201, 197, 202, 197, 197, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423474399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[1]R generation[0..12]: [109, 117, 113, 114, 113, 113, 118, 114, 118, 114, 114, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423488116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423501953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[2]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 201, 197, 202, 197, 197, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.42351554Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[2]R generation[0..12]: [109, 117, 113, 114, 113, 113, 118, 114, 118, 114, 114, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423529147Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423543044Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423555951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423568878Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423581465Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423594072Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"":"downstairs","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423609489Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423621516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"0","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423634663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"1","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:23:32.42364737Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"client":"2","":"downstairs","session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423664596Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423676623Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"707bec91-510a-4b58-97a5-0c5909085af7 is now active with session: a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423689311Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423700858Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664,"session_id":"a84ed3e5-9834-46e3-8b3f-9d4cc8d4707d"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:32.423712735Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 440 len 8192 data: 11 11 02/30 Read block 15 len 28672 03/30 Write block 139 len 36864 data: 8 9 8 10 10 13 14 17 15 04/30 Write block 454 len 16384 data: 8 6 5 5 05/30 Write block 475 len 32768 data: 9 10 8 5 8 8 10 9 06/30 Write block 335 len 12288 data: 10 8 6 07/30 Flush 08/30 Write block 162 len 16384 data: 12 12 10 11 09/30 Write block 304 len 40960 data: 8 5 6 7 7 8 7 7 7 6 10/30 Write block 523 len 28672 data: 16 16 15 15 11 11 11 11/30 Read block 541 len 8192 12/30 Read block 111 len 4096 13/30 Read block 119 len 36864 14/30 Flush 15/30 Write block 403 len 20480 data: 16 13 15 10 9 16/30 Flush 17/30 Write block 403 len 12288 data: 17 14 16 18/30 Read block 183 len 8192 19/30 Write block 128 len 16384 data: 7 7 8 9 20/30 Write block 177 len 20480 data: 12 11 11 12 13 21/30 Read block 32 len 4096 22/30 Read block 312 len 40960 23/30 Write block 380 len 20480 data: 11 9 9 9 10 24/30 Write block 386 len 20480 data: 11 11 13 12 12 25/30 Read block 237 len 36864 26/30 Write block 391 len 12288 data: 9 10 10 27/30 Read block 297 len 36864 28/30 Read block 500 len 32768 29/30 Write block 498 len 40960 data: 9 9 11 11 10 10 9 6 7 7 30/30 Read block 182 len 4096 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 14 12 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:32.442929503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1664} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 117 117 117 200 200 200 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 197 197 197 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 120 120 120 204 204 204 T T T 7 140-159 120 120 120 204 204 204 F F F 8 160-179 120 120 120 205 205 205 T T T 9 180-199 114 114 114 197 197 197 T T T 10 200-219 114 114 114 197 197 197 F F F 11 220-239 119 119 119 203 203 203 F F F 12 240-259 116 116 116 198 198 198 F F F 13 260-279 116 116 116 199 199 199 F F F 14 280-299 118 118 118 201 201 201 F F F 15 300-319 120 120 120 205 205 205 F F F 16 320-339 120 120 120 204 204 204 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 109 109 109 191 191 191 T T T 20 400-419 120 120 120 206 206 206 T T T 21 420-439 118 118 118 201 201 201 F F F 22 440-459 120 120 120 204 204 204 F F F 23 460-479 120 120 120 204 204 204 F F F 24 480-499 120 120 120 204 204 204 T T T 25 500-519 104 104 104 182 182 182 T T T 26 520-539 120 120 120 205 205 205 F F F 27 540-559 118 118 118 201 201 201 F F F 28 560-579 119 119 119 203 203 203 F F F 29 580-599 119 119 119 203 203 203 F F F Max gen: 120, Max flush: 206 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 191 191 191 F F F 1 020-039 117 117 117 200 200 200 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 197 197 197 F F F 4 080-099 113 113 113 196 196 196 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 121 121 121 207 207 207 F F F 7 140-159 120 120 120 204 204 204 F F F 8 160-179 121 121 121 207 207 207 F F F 9 180-199 121 121 121 207 207 207 F F F 10 200-219 114 114 114 197 197 197 F F F 11 220-239 119 119 119 203 203 203 F F F 12 240-259 116 116 116 198 198 198 F F F 13 260-279 116 116 116 199 199 199 F F F 14 280-299 118 118 118 201 201 201 F F F 15 300-319 120 120 120 205 205 205 F F F 16 320-339 120 120 120 204 204 204 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 110 110 110 192 192 192 F F F 19 380-399 121 121 121 207 207 207 F F F 20 400-419 121 121 121 207 207 207 F F F 21 420-439 118 118 118 201 201 201 F F F 22 440-459 120 120 120 204 204 204 F F F 23 460-479 120 120 120 204 204 204 F F F 24 480-499 121 121 121 207 207 207 F F F 25 500-519 121 121 121 207 207 207 F F F 26 520-539 120 120 120 205 205 205 F F F 27 540-559 118 118 118 201 201 201 F F F 28 560-579 119 119 119 203 203 203 F F F 29 580-599 119 119 119 203 203 203 F F F Max gen: 121, Max flush: 207 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.706102625Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.752504026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.753260865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.753304665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.753322181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.753345836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672} {"msg":"Crucible 470a0b5d-4bbe-4502-aac9-1d1f1f48c868 has session id: 8c777363-ee14-41c7-9683-3d48d3742db1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.753986601Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Upstairs opts: Upstairs UUID: 470a0b5d-4bbe-4502-aac9-1d1f1f48c868, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754099895Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Crucible stats registered with UUID: 470a0b5d-4bbe-4502-aac9-1d1f1f48c868","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754130678Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:36.754152163Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754749538Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754774542Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672} {"msg":"470a0b5d-4bbe-4502-aac9-1d1f1f48c868 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754842387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754860223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"0","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754882707Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"0","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754896444Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"1","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754911171Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"1","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754925188Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"2","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754939615Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"2","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754952592Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754965889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.754986834Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46121)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.755035403Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"ds_connection connected from Ok(127.0.0.1:42757)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.755065546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57995)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.755113715Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.755833502Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"1","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.755878392Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"2","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.755909655Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"0","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.756425168Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"1","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.756454662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"0","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.7566829Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"2","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.756997409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757081689Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[0]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 207, 204, 207, 207, 197, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757287223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[0]R generation[0..12]: [109, 117, 113, 114, 113, 113, 121, 120, 121, 121, 114, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757308698Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757323935Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[1]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 207, 204, 207, 207, 197, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757338062Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[1]R generation[0..12]: [109, 117, 113, 114, 113, 113, 121, 120, 121, 121, 114, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757351588Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757365455Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[2]R flush_numbers[0..12]: [191, 200, 196, 197, 196, 196, 207, 204, 207, 207, 197, 203]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757383671Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[2]R generation[0..12]: [109, 117, 113, 114, 113, 113, 121, 120, 121, 121, 114, 119]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757397578Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757412245Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757424952Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757437769Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757456984Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757469602Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"":"downstairs","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757482309Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757494996Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"0","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757508103Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"1","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.75752074Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"client":"2","":"downstairs","session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757533767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757545764Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"470a0b5d-4bbe-4502-aac9-1d1f1f48c868 is now active with session: 8c777363-ee14-41c7-9683-3d48d3742db1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757558302Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757570039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672,"session_id":"8c777363-ee14-41c7-9683-3d48d3742db1"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:36.757581946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 36864 data: 10 7 7 5 5 7 6 7 8 02/30 Read block 346 len 4096 03/30 Read block 72 len 8192 04/30 Read block 204 len 20480 05/30 Read block 560 len 4096 06/30 Write block 156 len 24576 data: 8 13 11 12 12 11 07/30 Read block 436 len 24576 08/30 Write block 407 len 36864 data: 10 8 6 8 10 9 9 9 10 09/30 Write block 375 len 40960 data: 12 12 14 12 12 12 10 10 10 11 10/30 Write block 94 len 24576 data: 11 13 13 14 13 11 11/30 Write block 25 len 28672 data: 6 9 9 10 9 10 10 12/30 Read block 465 len 32768 13/30 Read block 187 len 28672 14/30 Read block 132 len 24576 15/30 Write block 330 len 32768 data: 7 6 6 7 8 11 9 7 16/30 Write block 229 len 4096 data: 16 17/30 Read block 151 len 8192 18/30 Flush 19/30 Write block 301 len 32768 data: 3 5 7 9 6 7 8 8 20/30 Write block 254 len 28672 data: 6 6 8 7 8 9 9 21/30 Write block 155 len 32768 data: 7 9 14 12 13 13 12 13 22/30 Write block 14 len 8192 data: 5 5 23/30 Read block 488 len 24576 24/30 Read block 44 len 20480 25/30 Flush 26/30 Write block 239 len 36864 data: 9 10 13 13 12 10 10 12 9 27/30 Write block 64 len 28672 data: 11 12 12 11 12 14 14 28/30 Write block 315 len 8192 data: 9 10 29/30 Read block 523 len 40960 30/30 Read block 416 len 16384 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:36.777612609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1672} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 114 114 114 197 197 197 T T T 4 080-099 122 122 122 208 208 208 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 121 121 121 207 207 207 F F F 7 140-159 122 122 122 209 209 209 F F F 8 160-179 122 122 122 209 209 209 F F F 9 180-199 121 121 121 207 207 207 F F F 10 200-219 114 114 114 197 197 197 F F F 11 220-239 122 122 122 208 208 208 T T T 12 240-259 122 122 122 209 209 209 T T T 13 260-279 122 122 122 209 209 209 F F F 14 280-299 118 118 118 201 201 201 F F F 15 300-319 122 122 122 209 209 209 T T T 16 320-339 122 122 122 208 208 208 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 122 122 122 208 208 208 F F F 21 420-439 118 118 118 201 201 201 F F F 22 440-459 120 120 120 204 204 204 F F F 23 460-479 120 120 120 204 204 204 F F F 24 480-499 121 121 121 207 207 207 F F F 25 500-519 121 121 121 207 207 207 F F F 26 520-539 120 120 120 205 205 205 F F F 27 540-559 118 118 118 201 201 201 F F F 28 560-579 119 119 119 203 203 203 F F F 29 580-599 119 119 119 203 203 203 F F F Max gen: 122, Max flush: 209 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 123 123 123 210 210 210 F F F 4 080-099 122 122 122 208 208 208 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 121 121 121 207 207 207 F F F 7 140-159 122 122 122 209 209 209 F F F 8 160-179 122 122 122 209 209 209 F F F 9 180-199 121 121 121 207 207 207 F F F 10 200-219 114 114 114 197 197 197 F F F 11 220-239 123 123 123 210 210 210 F F F 12 240-259 123 123 123 210 210 210 F F F 13 260-279 122 122 122 209 209 209 F F F 14 280-299 118 118 118 201 201 201 F F F 15 300-319 123 123 123 210 210 210 F F F 16 320-339 122 122 122 208 208 208 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 122 122 122 208 208 208 F F F 21 420-439 118 118 118 201 201 201 F F F 22 440-459 120 120 120 204 204 204 F F F 23 460-479 120 120 120 204 204 204 F F F 24 480-499 121 121 121 207 207 207 F F F 25 500-519 121 121 121 207 207 207 F F F 26 520-539 120 120 120 205 205 205 F F F 27 540-559 118 118 118 201 201 201 F F F 28 560-579 119 119 119 203 203 203 F F F 29 580-599 119 119 119 203 203 203 F F F Max gen: 123, Max flush: 210 Begin loop 62 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.01557116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:41.059131365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.059492203Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.091707117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.092315469Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.0923564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.092378905Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.092397021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680} {"msg":"Crucible cbc02c3f-7e61-4a6f-ab56-82485af27e3d has session id: ad81ebce-9e2c-4d77-8982-070f0f0c25f4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093059011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Upstairs opts: Upstairs UUID: cbc02c3f-7e61-4a6f-ab56-82485af27e3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093180263Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Crucible stats registered with UUID: cbc02c3f-7e61-4a6f-ab56-82485af27e3d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093208677Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:41.093223454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093610156Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093643009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680} {"msg":"cbc02c3f-7e61-4a6f-ab56-82485af27e3d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093698896Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.0937265Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"0","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093752014Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"0","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.09376731Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"1","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093781877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"1","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093893322Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"2","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093932273Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"2","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.093963116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.09399025Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.094031211Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35507)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.094061814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56921)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.094110123Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:51656)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.094141206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.094700139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"1","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.094729292Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"2","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.09487383Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"0","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.095368838Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"2","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.095396931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"1","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.095431744Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"0","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.09597829Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096101382Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[0]R flush_numbers[0..12]: [209, 208, 196, 210, 208, 196, 207, 209, 209, 207, 197, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096170286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[0]R generation[0..12]: [122, 122, 113, 123, 122, 113, 121, 122, 122, 121, 114, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096191012Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096205908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[1]R flush_numbers[0..12]: [209, 208, 196, 210, 208, 196, 207, 209, 209, 207, 197, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096219785Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[1]R generation[0..12]: [122, 122, 113, 123, 122, 113, 121, 122, 122, 121, 114, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096233152Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096246839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[2]R flush_numbers[0..12]: [209, 208, 196, 210, 208, 196, 207, 209, 209, 207, 197, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096260536Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[2]R generation[0..12]: [122, 122, 113, 123, 122, 113, 121, 122, 122, 121, 114, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096273943Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.09628754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096300227Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096313114Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096325761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096338358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096357304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.09637271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"0","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096386267Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"1","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096399174Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"client":"2","":"downstairs","session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096411962Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096423849Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"cbc02c3f-7e61-4a6f-ab56-82485af27e3d is now active with session: ad81ebce-9e2c-4d77-8982-070f0f0c25f4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096436306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096447993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680,"session_id":"ad81ebce-9e2c-4d77-8982-070f0f0c25f4"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:41.096459891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 212 len 28672 data: 11 10 8 9 10 10 11 02/30 Write block 452 len 28672 data: 10 8 9 7 6 6 5 03/30 Write block 506 len 28672 data: 8 8 9 8 8 8 10 04/30 Read block 148 len 16384 05/30 Read block 508 len 8192 06/30 Write block 341 len 28672 data: 8 6 5 5 6 6 8 07/30 Read block 123 len 16384 08/30 Read block 215 len 32768 09/30 Read block 4 len 16384 10/30 Write block 334 len 20480 data: 9 12 10 8 8 11/30 Read block 312 len 20480 12/30 Flush 13/30 Write block 502 len 4096 data: 11 14/30 Read block 454 len 40960 15/30 Write block 142 len 16384 data: 11 11 14 15 16/30 Read block 192 len 32768 17/30 Read block 572 len 40960 18/30 Read block 364 len 24576 19/30 Read block 559 len 36864 20/30 Write block 156 len 36864 data: 10 15 13 14 14 13 14 13 11 21/30 Write block 256 len 20480 data: 9 8 9 10 10 22/30 Read block 347 len 4096 23/30 Read block 542 len 20480 24/30 Read block 35 len 28672 25/30 Read block 416 len 40960 26/30 Read block 141 len 32768 27/30 Write block 140 len 32768 data: 10 9 12 12 15 16 18 16 28/30 Write block 182 len 24576 data: 12 12 10 8 10 11 29/30 Write block 581 len 8192 data: 16 15 30/30 Read block 267 len 4096 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 1 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Done Sent Sent false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 3 3 6 Done 18 15 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:41.116747046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1680} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 123 123 123 210 210 210 F F F 4 080-099 122 122 122 208 208 208 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 121 121 121 207 207 207 F F F 7 140-159 122 122 122 209 209 209 T T T 8 160-179 122 122 122 209 209 209 T T T 9 180-199 121 121 121 207 207 207 T T T 10 200-219 124 124 124 211 211 211 F F F 11 220-239 123 123 123 210 210 210 F F F 12 240-259 123 123 123 210 210 210 T T T 13 260-279 122 122 122 209 209 209 T T T 14 280-299 118 118 118 201 201 201 F F F 15 300-319 123 123 123 210 210 210 F F F 16 320-339 124 124 124 211 211 211 F F F 17 340-359 124 124 124 211 211 211 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 122 122 122 208 208 208 F F F 21 420-439 118 118 118 201 201 201 F F F 22 440-459 124 124 124 211 211 211 F F F 23 460-479 120 120 120 204 204 204 F F F 24 480-499 121 121 121 207 207 207 F F F 25 500-519 124 124 124 211 211 211 T T T 26 520-539 120 120 120 205 205 205 F F F 27 540-559 118 118 118 201 201 201 F F F 28 560-579 119 119 119 203 203 203 F F F 29 580-599 119 119 119 203 203 203 T T T Max gen: 124, Max flush: 211 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 113 113 113 196 196 196 F F F 3 060-079 123 123 123 210 210 210 F F F 4 080-099 122 122 122 208 208 208 F F F 5 100-119 113 113 113 196 196 196 F F F 6 120-139 121 121 121 207 207 207 F F F 7 140-159 125 125 125 212 212 212 F F F 8 160-179 125 125 125 212 212 212 F F F 9 180-199 125 125 125 212 212 212 F F F 10 200-219 124 124 124 211 211 211 F F F 11 220-239 123 123 123 210 210 210 F F F 12 240-259 125 125 125 212 212 212 F F F 13 260-279 125 125 125 212 212 212 F F F 14 280-299 118 118 118 201 201 201 F F F 15 300-319 123 123 123 210 210 210 F F F 16 320-339 124 124 124 211 211 211 F F F 17 340-359 124 124 124 211 211 211 F F F 18 360-379 122 122 122 208 208 208 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 122 122 122 208 208 208 F F F 21 420-439 118 118 118 201 201 201 F F F 22 440-459 124 124 124 211 211 211 F F F 23 460-479 120 120 120 204 204 204 F F F 24 480-499 121 121 121 207 207 207 F F F 25 500-519 125 125 125 212 212 212 F F F 26 520-539 120 120 120 205 205 205 F F F 27 540-559 118 118 118 201 201 201 F F F 28 560-579 119 119 119 203 203 203 F F F 29 580-599 125 125 125 212 212 212 F F F Max gen: 125, Max flush: 212 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.401002433Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.446244196Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.446910765Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.446953435Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.446969572Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.446984168Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688} {"msg":"Crucible fa7e8e09-5a5b-4ea9-b94a-eed17219c31f has session id: dad01ee0-b52f-4ef6-af9a-d0fcf27453c2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.447618365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Upstairs opts: Upstairs UUID: fa7e8e09-5a5b-4ea9-b94a-eed17219c31f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.447746356Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Crucible stats registered with UUID: fa7e8e09-5a5b-4ea9-b94a-eed17219c31f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.44777046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:45.447783937Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448177028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.44821514Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688} {"msg":"fa7e8e09-5a5b-4ea9-b94a-eed17219c31f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448273627Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.44830101Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"0","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448326295Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"0","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448342561Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"1","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448398948Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"1","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448438169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"2","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448464943Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"2","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448491257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448518561Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448544805Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43424)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.44856922Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:33788)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448592534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41777)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.448616409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.449246416Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"1","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.449308172Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"0","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.449380096Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"2","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.449940839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"1","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450116719Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"0","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450143253Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"2","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450523367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450632462Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[0]R flush_numbers[0..12]: [209, 208, 196, 210, 208, 196, 207, 212, 212, 212, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450711804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[0]R generation[0..12]: [122, 122, 113, 123, 122, 113, 121, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.45073187Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450746426Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[1]R flush_numbers[0..12]: [209, 208, 196, 210, 208, 196, 207, 212, 212, 212, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450760073Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[1]R generation[0..12]: [122, 122, 113, 123, 122, 113, 121, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.45077364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450787367Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[2]R flush_numbers[0..12]: [209, 208, 196, 210, 208, 196, 207, 212, 212, 212, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450800904Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[2]R generation[0..12]: [122, 122, 113, 123, 122, 113, 121, 125, 125, 125, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.45081797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450832127Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450844964Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450858141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450871008Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO "session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450890664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"":"downstairs","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450903491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450915328Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"0","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450928305Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"1","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450941132Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"client":"2","":"downstairs","session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450954189Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450966117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"fa7e8e09-5a5b-4ea9-b94a-eed17219c31f is now active with session: dad01ee0-b52f-4ef6-af9a-d0fcf27453c2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450978584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.450990331Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688,"session_id":"dad01ee0-b52f-4ef6-af9a-d0fcf27453c2"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:45.451016835Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 525 len 8192 data: 16 16 02/30 Read block 429 len 16384 03/30 Read block 443 len 4096 04/30 Read block 453 len 20480 05/30 Write block 306 len 24576 data: 8 9 9 9 8 8 06/30 Read block 304 len 32768 07/30 Read block 52 len 40960 08/30 Write block 582 len 20480 data: 16 15 13 13 11 09/30 Write block 349 len 36864 data: 12 10 10 10 13 13 12 14 13 10/30 Read block 420 len 32768 11/30 Write block 288 len 28672 data: 5 7 7 6 6 5 4 12/30 Read block 206 len 36864 13/30 Read block 426 len 28672 14/30 Read block 260 len 20480 15/30 Write block 479 len 8192 data: 9 9 16/30 Read block 421 len 28672 17/30 Read block 336 len 28672 18/30 Flush 19/30 Write block 413 len 36864 data: 10 10 11 9 8 7 7 9 7 20/30 Read block 60 len 8192 21/30 Write block 80 len 32768 data: 8 11 12 12 11 8 8 9 22/30 Read block 412 len 16384 23/30 Write block 58 len 8192 data: 7 7 24/30 Write block 99 len 32768 data: 12 12 14 16 12 8 9 9 25/30 Write block 542 len 32768 data: 9 11 12 12 11 11 10 10 26/30 Write block 369 len 12288 data: 7 8 7 27/30 Write block 360 len 28672 data: 9 11 9 10 13 13 9 28/30 Write block 279 len 4096 data: 9 29/30 Read block 596 len 12288 30/30 Write block 186 len 20480 data: 11 12 11 12 12 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:45.470779769Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1688} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 113 113 113 196 196 196 T T T 3 060-079 123 123 123 210 210 210 F F F 4 080-099 122 122 122 208 208 208 T T T 5 100-119 113 113 113 196 196 196 T T T 6 120-139 121 121 121 207 207 207 F F F 7 140-159 125 125 125 212 212 212 F F F 8 160-179 125 125 125 212 212 212 F F F 9 180-199 125 125 125 212 212 212 T T T 10 200-219 124 124 124 211 211 211 F F F 11 220-239 123 123 123 210 210 210 F F F 12 240-259 125 125 125 212 212 212 F F F 13 260-279 125 125 125 212 212 212 T T T 14 280-299 126 126 126 213 213 213 F F F 15 300-319 126 126 126 213 213 213 F F F 16 320-339 124 124 124 211 211 211 F F F 17 340-359 126 126 126 213 213 213 F F F 18 360-379 122 122 122 208 208 208 T T T 19 380-399 122 122 122 208 208 208 F F F 20 400-419 122 122 122 208 208 208 T T T 21 420-439 118 118 118 201 201 201 T T T 22 440-459 124 124 124 211 211 211 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 126 126 126 213 213 213 F F F 25 500-519 125 125 125 212 212 212 F F F 26 520-539 126 126 126 213 213 213 F F F 27 540-559 118 118 118 201 201 201 T T T 28 560-579 119 119 119 203 203 203 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 126, Max flush: 213 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 127 127 127 214 214 214 F F F 3 060-079 123 123 123 210 210 210 F F F 4 080-099 127 127 127 214 214 214 F F F 5 100-119 127 127 127 214 214 214 F F F 6 120-139 121 121 121 207 207 207 F F F 7 140-159 125 125 125 212 212 212 F F F 8 160-179 125 125 125 212 212 212 F F F 9 180-199 127 127 127 214 214 214 F F F 10 200-219 124 124 124 211 211 211 F F F 11 220-239 123 123 123 210 210 210 F F F 12 240-259 125 125 125 212 212 212 F F F 13 260-279 127 127 127 214 214 214 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 126 126 126 213 213 213 F F F 16 320-339 124 124 124 211 211 211 F F F 17 340-359 126 126 126 213 213 213 F F F 18 360-379 127 127 127 214 214 214 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 127 127 127 214 214 214 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 124 124 124 211 211 211 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 126 126 126 213 213 213 F F F 25 500-519 125 125 125 212 212 212 F F F 26 520-539 126 126 126 213 213 213 F F F 27 540-559 127 127 127 214 214 214 F F F 28 560-579 119 119 119 203 203 203 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 127, Max flush: 214 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.735480196Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:49.779714637Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.780189569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.812605258Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.813310268Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.813358447Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.813382142Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.813400458Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696} {"msg":"Crucible ae4219d7-88f0-4503-8263-3052ef3d9e03 has session id: 07bf1443-4960-40f1-9e7f-409e800591d0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.813997302Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Upstairs opts: Upstairs UUID: ae4219d7-88f0-4503-8263-3052ef3d9e03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814047361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Crucible stats registered with UUID: ae4219d7-88f0-4503-8263-3052ef3d9e03","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814073965Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:49.81409476Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814521344Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814543569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696} {"msg":"ae4219d7-88f0-4503-8263-3052ef3d9e03 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814597727Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814623741Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"0","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814647295Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"0","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814661382Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"1","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814675789Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"1","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814689126Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"2","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814707292Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"2","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814721389Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814744703Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814768238Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34567)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814887521Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:45391)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814929761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"ds_connection connected from Ok(127.0.0.1:48571)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.814965363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.815602819Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"1","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.81564342Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"2","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.815762713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"0","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.816233056Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"2","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.81626204Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"1","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.816380733Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"0","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.816761787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.81678889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[0]R flush_numbers[0..12]: [209, 208, 214, 210, 214, 214, 207, 212, 212, 214, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.816950894Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[0]R generation[0..12]: [122, 122, 127, 123, 127, 127, 121, 125, 125, 127, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.816972669Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.816987555Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[1]R flush_numbers[0..12]: [209, 208, 214, 210, 214, 214, 207, 212, 212, 214, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817001432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[1]R generation[0..12]: [122, 122, 127, 123, 127, 127, 121, 125, 125, 127, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817015559Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817029876Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[2]R flush_numbers[0..12]: [209, 208, 214, 210, 214, 214, 207, 212, 212, 214, 211, 210]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817043523Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[2]R generation[0..12]: [122, 122, 127, 123, 127, 127, 121, 125, 125, 127, 124, 123]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.8170569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817070606Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817083364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817096401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817108838Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817121295Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"":"downstairs","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817133952Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817148519Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"0","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817161766Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"1","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817181352Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"client":"2","":"downstairs","session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817194438Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817206306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"ae4219d7-88f0-4503-8263-3052ef3d9e03 is now active with session: 07bf1443-4960-40f1-9e7f-409e800591d0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817218543Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.81723027Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696,"session_id":"07bf1443-4960-40f1-9e7f-409e800591d0"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:49.817242418Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 333 len 28672 data: 8 10 13 11 9 9 9 02/30 Write block 316 len 28672 data: 11 10 9 10 11 14 13 03/30 Write block 51 len 24576 data: 7 7 8 9 8 8 04/30 Read block 478 len 8192 05/30 Flush 06/30 Write block 119 len 28672 data: 10 10 7 7 7 6 5 07/30 Flush 08/30 Write block 351 len 40960 data: 11 11 14 14 13 15 14 11 9 10 09/30 Write block 126 len 8192 data: 6 7 10/30 Write block 96 len 40960 data: 14 15 14 13 13 15 17 13 9 10 11/30 Write block 50 len 20480 data: 7 8 8 9 10 12/30 Read block 371 len 24576 13/30 Flush 14/30 Write block 227 len 32768 data: 17 16 17 16 16 14 13 13 15/30 Read block 309 len 12288 16/30 Flush 17/30 Write block 233 len 20480 data: 14 14 13 12 12 18/30 Write block 218 len 32768 data: 12 10 9 10 14 19 17 18 19/30 Write block 104 len 8192 data: 10 11 20/30 Flush 21/30 Write block 189 len 16384 data: 13 13 13 13 22/30 Write block 245 len 4096 data: 11 23/30 Read block 418 len 40960 24/30 Write block 176 len 8192 data: 10 13 25/30 Read block 246 len 40960 26/30 Read block 38 len 12288 27/30 Write block 50 len 32768 data: 8 9 9 10 11 9 9 7 28/30 Write block 557 len 16384 data: 5 5 5 7 29/30 Write block 163 len 4096 data: 14 30/30 Read block 556 len 4096 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:49.835208429Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1696} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 128 128 128 217 217 217 T T T 3 060-079 123 123 123 210 210 210 F F F 4 080-099 128 128 128 217 217 217 F F F 5 100-119 128 128 128 219 219 219 F F F 6 120-139 128 128 128 217 217 217 F F F 7 140-159 125 125 125 212 212 212 F F F 8 160-179 125 125 125 212 212 212 T T T 9 180-199 127 127 127 214 214 214 T T T 10 200-219 128 128 128 219 219 219 F F F 11 220-239 128 128 128 219 219 219 F F F 12 240-259 125 125 125 212 212 212 T T T 13 260-279 127 127 127 214 214 214 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 128 128 128 215 215 215 F F F 16 320-339 128 128 128 215 215 215 F F F 17 340-359 128 128 128 217 217 217 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 127 127 127 214 214 214 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 124 124 124 211 211 211 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 126 126 126 213 213 213 F F F 25 500-519 125 125 125 212 212 212 F F F 26 520-539 126 126 126 213 213 213 F F F 27 540-559 127 127 127 214 214 214 T T T 28 560-579 119 119 119 203 203 203 T T T 29 580-599 126 126 126 213 213 213 F F F Max gen: 128, Max flush: 219 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 122 122 122 208 208 208 F F F 2 040-059 129 129 129 220 220 220 F F F 3 060-079 123 123 123 210 210 210 F F F 4 080-099 128 128 128 217 217 217 F F F 5 100-119 128 128 128 219 219 219 F F F 6 120-139 128 128 128 217 217 217 F F F 7 140-159 125 125 125 212 212 212 F F F 8 160-179 129 129 129 220 220 220 F F F 9 180-199 129 129 129 220 220 220 F F F 10 200-219 128 128 128 219 219 219 F F F 11 220-239 128 128 128 219 219 219 F F F 12 240-259 129 129 129 220 220 220 F F F 13 260-279 127 127 127 214 214 214 F F F 14 280-299 126 126 126 213 213 213 F F F 15 300-319 128 128 128 215 215 215 F F F 16 320-339 128 128 128 215 215 215 F F F 17 340-359 128 128 128 217 217 217 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 122 122 122 208 208 208 F F F 20 400-419 127 127 127 214 214 214 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 124 124 124 211 211 211 F F F 23 460-479 126 126 126 213 213 213 F F F 24 480-499 126 126 126 213 213 213 F F F 25 500-519 125 125 125 212 212 212 F F F 26 520-539 126 126 126 213 213 213 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 129 129 129 220 220 220 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 129, Max flush: 220 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.118714256Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.164798059Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.1655011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.165547319Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.165562406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.165576873Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704} {"msg":"Crucible 757674f8-d583-460a-b123-687d9ee152b4 has session id: 0c635dd1-724f-42f6-9562-db6af295b66b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166196372Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Upstairs opts: Upstairs UUID: 757674f8-d583-460a-b123-687d9ee152b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166237333Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Crucible stats registered with UUID: 757674f8-d583-460a-b123-687d9ee152b4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166253819Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:54.166266496Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166702888Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.16673658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704} {"msg":"757674f8-d583-460a-b123-687d9ee152b4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166788468Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166820391Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"0","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166845605Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"0","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166860132Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"1","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166874189Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"1","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166896484Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"2","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166922788Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"2","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166946913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166966408Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.166991472Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64240)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.167168722Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49915)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.167234697Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38536)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.167269839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.167862265Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"0","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.167898707Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"1","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.168113308Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"2","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.168664073Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"1","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.168692377Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"0","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.168707494Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"2","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169231015Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.16925548Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[0]R flush_numbers[0..12]: [209, 208, 220, 210, 217, 219, 217, 212, 220, 220, 219, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169328183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[0]R generation[0..12]: [122, 122, 129, 123, 128, 128, 128, 125, 129, 129, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169348249Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169362875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[1]R flush_numbers[0..12]: [209, 208, 220, 210, 217, 219, 217, 212, 220, 220, 219, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169376822Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[1]R generation[0..12]: [122, 122, 129, 123, 128, 128, 128, 125, 129, 129, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169390749Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169404446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[2]R flush_numbers[0..12]: [209, 208, 220, 210, 217, 219, 217, 212, 220, 220, 219, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169417853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[2]R generation[0..12]: [122, 122, 129, 123, 128, 128, 128, 125, 129, 129, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169432839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169450685Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169463883Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169476809Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169489307Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169501904Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"":"downstairs","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:23:54.169514671Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169531867Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"0","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169544874Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"1","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169557501Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"client":"2","":"downstairs","session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169570278Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169582176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"757674f8-d583-460a-b123-687d9ee152b4 is now active with session: 0c635dd1-724f-42f6-9562-db6af295b66b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169594723Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.16960631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704,"session_id":"0c635dd1-724f-42f6-9562-db6af295b66b"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:54.169618368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 458 len 24576 data: 6 7 5 6 7 8 02/30 Read block 575 len 32768 03/30 Flush 04/30 Write block 289 len 12288 data: 8 8 7 05/30 Write block 39 len 36864 data: 10 11 11 10 14 14 15 11 9 06/30 Write block 155 len 36864 data: 8 11 16 14 15 15 14 15 15 07/30 Write block 567 len 28672 data: 7 10 12 12 12 13 11 08/30 Write block 443 len 32768 data: 12 13 12 10 12 11 12 13 09/30 Read block 15 len 24576 10/30 Flush 11/30 Write block 186 len 36864 data: 12 13 12 14 14 14 14 10 12 12/30 Read block 511 len 4096 13/30 Read block 329 len 36864 14/30 Read block 313 len 40960 15/30 Read block 407 len 28672 16/30 Write block 103 len 20480 data: 14 11 12 10 11 17/30 Write block 195 len 24576 data: 10 9 8 9 10 10 18/30 Write block 247 len 32768 data: 10 11 11 9 11 8 8 7 19/30 Write block 73 len 28672 data: 8 8 5 5 6 5 7 20/30 Write block 497 len 36864 data: 8 10 10 12 12 12 11 10 7 21/30 Flush 22/30 Write block 65 len 20480 data: 13 13 12 13 15 23/30 Flush 24/30 Write block 397 len 24576 data: 7 10 9 10 12 16 25/30 Read block 537 len 4096 26/30 Write block 81 len 40960 data: 12 13 13 12 9 9 10 10 11 11 27/30 Write block 154 len 32768 data: 6 9 12 17 15 16 16 15 28/30 Read block 278 len 36864 29/30 Write block 571 len 20480 data: 13 14 12 9 10 30/30 Write block 187 len 8192 data: 14 13 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:54.189336722Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1704} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 130 130 130 222 222 222 F F F 2 040-059 130 130 130 222 222 222 F F F 3 060-079 130 130 130 224 224 224 F F F 4 080-099 128 128 128 217 217 217 T T T 5 100-119 130 130 130 223 223 223 F F F 6 120-139 128 128 128 217 217 217 F F F 7 140-159 130 130 130 222 222 222 T T T 8 160-179 130 130 130 222 222 222 T T T 9 180-199 130 130 130 223 223 223 T T T 10 200-219 130 130 130 223 223 223 F F F 11 220-239 128 128 128 219 219 219 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 127 127 127 214 214 214 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 128 128 128 215 215 215 F F F 16 320-339 128 128 128 215 215 215 F F F 17 340-359 128 128 128 217 217 217 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 122 122 122 208 208 208 T T T 20 400-419 127 127 127 214 214 214 T T T 21 420-439 127 127 127 214 214 214 F F F 22 440-459 130 130 130 222 222 222 F F F 23 460-479 130 130 130 221 221 221 F F F 24 480-499 130 130 130 223 223 223 F F F 25 500-519 130 130 130 223 223 223 F F F 26 520-539 126 126 126 213 213 213 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 130 130 130 222 222 222 T T T 29 580-599 126 126 126 213 213 213 F F F Max gen: 130, Max flush: 224 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 130 130 130 222 222 222 F F F 2 040-059 130 130 130 222 222 222 F F F 3 060-079 130 130 130 224 224 224 F F F 4 080-099 131 131 131 225 225 225 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 128 128 128 217 217 217 F F F 7 140-159 131 131 131 225 225 225 F F F 8 160-179 131 131 131 225 225 225 F F F 9 180-199 131 131 131 225 225 225 F F F 10 200-219 130 130 130 223 223 223 F F F 11 220-239 128 128 128 219 219 219 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 127 127 127 214 214 214 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 128 128 128 215 215 215 F F F 16 320-339 128 128 128 215 215 215 F F F 17 340-359 128 128 128 217 217 217 F F F 18 360-379 128 128 128 217 217 217 F F F 19 380-399 131 131 131 225 225 225 F F F 20 400-419 131 131 131 225 225 225 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 130 130 130 222 222 222 F F F 23 460-479 130 130 130 221 221 221 F F F 24 480-499 130 130 130 223 223 223 F F F 25 500-519 130 130 130 223 223 223 F F F 26 520-539 126 126 126 213 213 213 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 131 131 131 225 225 225 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 131, Max flush: 225 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.443347249Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.489011877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.489733484Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.489788431Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.489806917Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.489824773Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712} {"msg":"Crucible 82d32eae-543e-4144-b9cb-c1a2ef85813f has session id: f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.490369469Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Upstairs opts: Upstairs UUID: 82d32eae-543e-4144-b9cb-c1a2ef85813f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.4904117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Crucible stats registered with UUID: 82d32eae-543e-4144-b9cb-c1a2ef85813f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.490428946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:58.490443523Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.49089906Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.490925324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712} {"msg":"82d32eae-543e-4144-b9cb-c1a2ef85813f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.490990869Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491009855Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"0","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491032869Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"0","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491046786Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"1","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491061663Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"1","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491076689Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"2","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491090466Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"2","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491105013Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.49111934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491133436Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:42139)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491463792Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59249)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491501723Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54096)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.491525458Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.492086691Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"2","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.492114984Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"0","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.492137699Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"1","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.492712759Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"1","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.492764777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"2","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.492834651Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"0","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493305155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493433006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[0]R flush_numbers[0..12]: [209, 222, 222, 224, 225, 223, 217, 225, 225, 225, 223, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493472197Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[0]R generation[0..12]: [122, 130, 130, 130, 131, 130, 128, 131, 131, 131, 130, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493492202Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493507419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[1]R flush_numbers[0..12]: [209, 222, 222, 224, 225, 223, 217, 225, 225, 225, 223, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493524935Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[1]R generation[0..12]: [122, 130, 130, 130, 131, 130, 128, 131, 131, 131, 130, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493539482Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493553768Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[2]R flush_numbers[0..12]: [209, 222, 222, 224, 225, 223, 217, 225, 225, 225, 223, 219]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493567325Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[2]R generation[0..12]: [122, 130, 130, 130, 131, 130, 128, 131, 131, 131, 130, 128]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493580762Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493594339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493607106Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493621993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493644498Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493672821Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"":"downstairs","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493695116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493708403Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"0","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.49372193Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"1","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493734927Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"client":"2","":"downstairs","session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493750674Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493763021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"82d32eae-543e-4144-b9cb-c1a2ef85813f is now active with session: f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493775658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493787286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712,"session_id":"f44e34ee-9b8d-4ec0-b0b1-13e0dab31c7d"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:23:58.493799333Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 60 len 32768 data: 7 10 9 11 12 14 14 13 02/30 Read block 142 len 24576 03/30 Flush 04/30 Write block 339 len 24576 data: 10 9 9 7 6 6 05/30 Write block 85 len 24576 data: 10 10 11 11 12 12 06/30 Flush 07/30 Write block 173 len 24576 data: 11 11 10 11 14 12 08/30 Write block 328 len 20480 data: 8 8 8 7 7 09/30 Read block 414 len 8192 10/30 Write block 334 len 28672 data: 11 14 12 10 10 11 10 11/30 Read block 229 len 28672 12/30 Flush 13/30 Write block 523 len 28672 data: 17 17 17 17 12 12 12 14/30 Read block 200 len 36864 15/30 Write block 41 len 4096 data: 12 16/30 Read block 446 len 28672 17/30 Write block 390 len 8192 data: 13 10 18/30 Read block 160 len 32768 19/30 Write block 171 len 32768 data: 10 11 12 12 11 12 15 13 20/30 Read block 305 len 28672 21/30 Write block 361 len 8192 data: 12 10 22/30 Write block 476 len 32768 data: 11 9 6 10 10 11 10 8 23/30 Read block 490 len 16384 24/30 Read block 436 len 4096 25/30 Write block 313 len 36864 data: 7 8 10 12 11 10 11 12 15 26/30 Read block 573 len 32768 27/30 Write block 216 len 40960 data: 11 11 13 11 10 11 15 20 18 19 28/30 Write block 216 len 32768 data: 12 12 14 12 11 12 16 21 29/30 Write block 75 len 40960 data: 6 6 7 6 8 9 13 14 14 13 30/30 Read block 519 len 40960 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 7 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 0 2 Done 18 16 18 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:23:58.514236244Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1712} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 130 130 130 222 222 222 F F F 2 040-059 130 130 130 222 222 222 T T T 3 060-079 132 132 132 226 226 226 T T T 4 080-099 132 132 132 227 227 227 T T T 5 100-119 130 130 130 223 223 223 F F F 6 120-139 128 128 128 217 217 217 F F F 7 140-159 131 131 131 225 225 225 F F F 8 160-179 132 132 132 228 228 228 T T T 9 180-199 131 131 131 225 225 225 F F F 10 200-219 130 130 130 223 223 223 T T T 11 220-239 128 128 128 219 219 219 T T T 12 240-259 130 130 130 223 223 223 F F F 13 260-279 127 127 127 214 214 214 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 128 128 128 215 215 215 T T T 16 320-339 132 132 132 228 228 228 T T T 17 340-359 132 132 132 228 228 228 F F F 18 360-379 128 128 128 217 217 217 T T T 19 380-399 131 131 131 225 225 225 T T T 20 400-419 131 131 131 225 225 225 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 130 130 130 222 222 222 F F F 23 460-479 130 130 130 221 221 221 T T T 24 480-499 130 130 130 223 223 223 T T T 25 500-519 130 130 130 223 223 223 F F F 26 520-539 126 126 126 213 213 213 T T T 27 540-559 129 129 129 220 220 220 F F F 28 560-579 131 131 131 225 225 225 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 132, Max flush: 228 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 F F F 1 020-039 130 130 130 222 222 222 F F F 2 040-059 133 133 133 229 229 229 F F F 3 060-079 133 133 133 229 229 229 F F F 4 080-099 133 133 133 229 229 229 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 128 128 128 217 217 217 F F F 7 140-159 131 131 131 225 225 225 F F F 8 160-179 133 133 133 229 229 229 F F F 9 180-199 131 131 131 225 225 225 F F F 10 200-219 133 133 133 229 229 229 F F F 11 220-239 133 133 133 229 229 229 F F F 12 240-259 130 130 130 223 223 223 F F F 13 260-279 127 127 127 214 214 214 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 133 133 133 229 229 229 F F F 16 320-339 133 133 133 229 229 229 F F F 17 340-359 132 132 132 228 228 228 F F F 18 360-379 133 133 133 229 229 229 F F F 19 380-399 133 133 133 229 229 229 F F F 20 400-419 131 131 131 225 225 225 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 130 130 130 222 222 222 F F F 23 460-479 133 133 133 229 229 229 F F F 24 480-499 133 133 133 229 229 229 F F F 25 500-519 130 130 130 223 223 223 F F F 26 520-539 133 133 133 229 229 229 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 131 131 131 225 225 225 F F F 29 580-599 126 126 126 213 213 213 F F F Max gen: 133, Max flush: 229 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.806763357Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.854805537Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.855467167Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.855519945Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.85553929Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.855557336Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720} {"msg":"Crucible 466ee054-863d-447d-a488-08bf1c0cbaa4 has session id: fb0f4dfa-f752-41d2-adf0-ac1902d95c98","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856197751Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Upstairs opts: Upstairs UUID: 466ee054-863d-447d-a488-08bf1c0cbaa4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856238272Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Crucible stats registered with UUID: 466ee054-863d-447d-a488-08bf1c0cbaa4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856255348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:02.856268745Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856782729Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856807613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720} {"msg":"466ee054-863d-447d-a488-08bf1c0cbaa4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856848224Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.85686396Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"0","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856894863Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"0","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.85691176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"1","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856926766Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"1","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856940893Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"2","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.85695523Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"2","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856968456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.856982224Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.857007718Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"ds_connection connected from Ok(127.0.0.1:61223)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.857167991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"ds_connection connected from Ok(127.0.0.1:54390)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.857207393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"ds_connection connected from Ok(127.0.0.1:57583)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.857236286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.857891487Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"2","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.857932568Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"0","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.858520115Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"0","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.858704823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"2","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.858836704Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"1","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.8590292Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859308347Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859387859Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"1","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[0]R flush_numbers[0..12]: [209, 222, 229, 229, 229, 223, 217, 225, 229, 225, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859881777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[0]R generation[0..12]: [122, 130, 133, 133, 133, 130, 128, 131, 133, 131, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859901942Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859917259Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[1]R flush_numbers[0..12]: [209, 222, 229, 229, 229, 223, 217, 225, 229, 225, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859931036Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[1]R generation[0..12]: [122, 130, 133, 133, 133, 130, 128, 131, 133, 131, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859944622Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859958419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[2]R flush_numbers[0..12]: [209, 222, 229, 229, 229, 223, 217, 225, 229, 225, 229, 229]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859971956Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[2]R generation[0..12]: [122, 130, 133, 133, 133, 130, 128, 131, 133, 131, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.859985833Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.86000027Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860017936Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860032093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Next flush: 230","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.86004488Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860057507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"":"downstairs","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860070424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,Wait for a query_work_queue command to finish before sending IO "session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.86008931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"0","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860102497Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"1","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860115644Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"client":"2","":"downstairs","session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860128791Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860140898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"466ee054-863d-447d-a488-08bf1c0cbaa4 is now active with session: fb0f4dfa-f752-41d2-adf0-ac1902d95c98","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860153445Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.860165093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720,"session_id":"fb0f4dfa-f752-41d2-adf0-ac1902d95c98"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:02.86017757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 591 len 8192 data: 8 7 02/30 Write block 227 len 36864 data: 18 17 18 17 17 15 15 15 14 03/30 Read block 42 len 12288 04/30 Read block 292 len 20480 05/30 Read block 195 len 8192 06/30 Write block 362 len 40960 data: 11 11 14 14 10 9 10 8 9 8 07/30 Write block 461 len 40960 data: 7 8 9 9 9 11 11 12 10 9 08/30 Flush 09/30 Write block 301 len 16384 data: 4 6 8 10 10/30 Read block 408 len 24576 11/30 Flush 12/30 Write block 458 len 36864 data: 7 8 6 8 9 10 10 10 12 13/30 Write block 163 len 32768 data: 16 12 12 11 11 9 11 9 14/30 Write block 254 len 36864 data: 8 7 10 9 10 11 11 12 9 15/30 Read block 394 len 40960 16/30 Write block 583 len 24576 data: 16 14 14 12 11 8 17/30 Flush 18/30 Flush 19/30 Write block 132 len 28672 data: 10 9 7 7 10 7 9 20/30 Flush 21/30 Flush 22/30 Write block 66 len 4096 data: 15 23/30 Read block 243 len 24576 24/30 Read block 475 len 32768 25/30 Flush 26/30 Write block 499 len 12288 data: 11 13 13 27/30 Read block 448 len 20480 28/30 Read block 541 len 36864 29/30 Read block 348 len 4096 30/30 Write block 19 len 12288 data: 7 9 10 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:02.88075127Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1720} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 122 122 122 209 209 209 T T T 1 020-039 130 130 130 222 222 222 T T T 2 040-059 133 133 133 229 229 229 F F F 3 060-079 134 134 134 236 236 236 F F F 4 080-099 133 133 133 229 229 229 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 134 134 134 234 234 234 F F F 7 140-159 131 131 131 225 225 225 F F F 8 160-179 134 134 134 232 232 232 F F F 9 180-199 131 131 131 225 225 225 F F F 10 200-219 133 133 133 229 229 229 F F F 11 220-239 134 134 134 230 230 230 F F F 12 240-259 134 134 134 232 232 232 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 134 134 134 231 231 231 F F F 16 320-339 133 133 133 229 229 229 F F F 17 340-359 132 132 132 228 228 228 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 133 133 133 229 229 229 F F F 20 400-419 131 131 131 225 225 225 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 134 134 134 232 232 232 F F F 24 480-499 133 133 133 229 229 229 T T T 25 500-519 130 130 130 223 223 223 T T T 26 520-539 133 133 133 229 229 229 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 131 131 131 225 225 225 F F F 29 580-599 134 134 134 232 232 232 F F F Max gen: 134, Max flush: 236 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 237 237 237 F F F 1 020-039 135 135 135 237 237 237 F F F 2 040-059 133 133 133 229 229 229 F F F 3 060-079 134 134 134 236 236 236 F F F 4 080-099 133 133 133 229 229 229 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 134 134 134 234 234 234 F F F 7 140-159 131 131 131 225 225 225 F F F 8 160-179 134 134 134 232 232 232 F F F 9 180-199 131 131 131 225 225 225 F F F 10 200-219 133 133 133 229 229 229 F F F 11 220-239 134 134 134 230 230 230 F F F 12 240-259 134 134 134 232 232 232 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 134 134 134 231 231 231 F F F 16 320-339 133 133 133 229 229 229 F F F 17 340-359 132 132 132 228 228 228 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 133 133 133 229 229 229 F F F 20 400-419 131 131 131 225 225 225 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 134 134 134 232 232 232 F F F 24 480-499 135 135 135 237 237 237 F F F 25 500-519 135 135 135 237 237 237 F F F 26 520-539 133 133 133 229 229 229 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 131 131 131 225 225 225 F F F 29 580-599 134 134 134 232 232 232 F F F Max gen: 135, Max flush: 237 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.12013673Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.165655101Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.1663656Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.166416469Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.166434604Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.166452531Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728} {"msg":"Crucible 4328d6a2-9775-45cb-9d40-991fa757d076 has session id: dc8fb29c-5078-415b-a4cc-59885b5c6848","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.166995637Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Upstairs opts: Upstairs UUID: 4328d6a2-9775-45cb-9d40-991fa757d076, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167054504Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Crucible stats registered with UUID: 4328d6a2-9775-45cb-9d40-991fa757d076","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167079018Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:07.167104293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167703407Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167730191Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728} {"msg":"4328d6a2-9775-45cb-9d40-991fa757d076 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167751966Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167765183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"0","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167786009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"0","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167799865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"1","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167814732Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"1","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167827549Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"2","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167841256Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"2","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167854183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.167874229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.168216181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"ds_connection connected from Ok(127.0.0.1:58585)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.168259531Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"ds_connection connected from Ok(127.0.0.1:51854)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.168288944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"ds_connection connected from Ok(127.0.0.1:40391)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.168319908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.168817245Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"0","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.168858706Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"1","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.168953974Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"2","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.169481185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"1","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.169509858Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"0","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.169585401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"2","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170101474Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.17020686Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[0]R flush_numbers[0..12]: [237, 237, 229, 236, 229, 223, 234, 225, 232, 225, 229, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170267047Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[0]R generation[0..12]: [135, 135, 133, 134, 133, 130, 134, 131, 134, 131, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170286552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170301129Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[1]R flush_numbers[0..12]: [237, 237, 229, 236, 229, 223, 234, 225, 232, 225, 229, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170315206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[1]R generation[0..12]: [135, 135, 133, 134, 133, 130, 134, 131, 134, 131, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170328793Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170346099Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[2]R flush_numbers[0..12]: [237, 237, 229, 236, 229, 223, 234, 225, 232, 225, 229, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170360006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[2]R generation[0..12]: [135, 135, 133, 134, 133, 130, 134, 131, 134, 131, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170373452Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170387099Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170399766Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170412754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170425321Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170438008Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"":"downstairs","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170500254Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.17051765Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"0","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170531417Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"1","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170544354Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"client":"2","":"downstairs","session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170557121Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170569108Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"4328d6a2-9775-45cb-9d40-991fa757d076 is now active with session: dc8fb29c-5078-415b-a4cc-59885b5c6848","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170581655Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170597492Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728,"session_id":"dc8fb29c-5078-415b-a4cc-59885b5c6848"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:07.170610109Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 523 len 8192 data: 18 18 02/30 Write block 19 len 32768 data: 8 10 11 7 6 5 7 10 03/30 Flush 04/30 Flush 05/30 Write block 571 len 36864 data: 14 15 13 10 11 12 14 13 16 06/30 Flush 07/30 Write block 389 len 16384 data: 13 14 11 11 08/30 Write block 520 len 4096 data: 16 09/30 Write block 13 len 28672 data: 6 6 6 8 6 6 9 10/30 Read block 273 len 8192 11/30 Read block 139 len 40960 12/30 Read block 119 len 12288 13/30 Read block 555 len 12288 14/30 Write block 197 len 12288 data: 9 10 11 15/30 Write block 303 len 8192 data: 9 11 16/30 Read block 561 len 12288 17/30 Write block 496 len 32768 data: 8 9 11 12 14 14 13 12 18/30 Read block 473 len 28672 19/30 Read block 337 len 32768 20/30 Read block 143 len 24576 21/30 Write block 467 len 36864 data: 12 13 11 10 10 12 11 10 10 22/30 Write block 81 len 24576 data: 14 15 15 14 11 11 23/30 Flush 24/30 Write block 479 len 12288 data: 11 11 12 25/30 Read block 182 len 40960 26/30 Read block 297 len 16384 27/30 Write block 138 len 20480 data: 10 9 11 10 13 28/30 Read block 461 len 8192 29/30 Read block 528 len 12288 30/30 Read block 36 len 32768 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:07.190557841Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1728} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 133 133 133 229 229 229 F F F 3 060-079 134 134 134 236 236 236 F F F 4 080-099 136 136 136 241 241 241 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 134 134 134 234 234 234 T T T 7 140-159 131 131 131 225 225 225 T T T 8 160-179 134 134 134 232 232 232 F F F 9 180-199 136 136 136 241 241 241 F F F 10 200-219 133 133 133 229 229 229 F F F 11 220-239 134 134 134 230 230 230 F F F 12 240-259 134 134 134 232 232 232 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 136 136 136 241 241 241 F F F 16 320-339 133 133 133 229 229 229 F F F 17 340-359 132 132 132 228 228 228 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 136 136 136 241 241 241 F F F 20 400-419 131 131 131 225 225 225 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 136 136 136 241 241 241 T T T 24 480-499 136 136 136 241 241 241 T T T 25 500-519 136 136 136 241 241 241 F F F 26 520-539 136 136 136 241 241 241 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 136 136 136 240 240 240 F F F 29 580-599 134 134 134 232 232 232 F F F Max gen: 136, Max flush: 241 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 133 133 133 229 229 229 F F F 3 060-079 134 134 134 236 236 236 F F F 4 080-099 136 136 136 241 241 241 F F F 5 100-119 130 130 130 223 223 223 F F F 6 120-139 137 137 137 242 242 242 F F F 7 140-159 137 137 137 242 242 242 F F F 8 160-179 134 134 134 232 232 232 F F F 9 180-199 136 136 136 241 241 241 F F F 10 200-219 133 133 133 229 229 229 F F F 11 220-239 134 134 134 230 230 230 F F F 12 240-259 134 134 134 232 232 232 F F F 13 260-279 134 134 134 232 232 232 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 136 136 136 241 241 241 F F F 16 320-339 133 133 133 229 229 229 F F F 17 340-359 132 132 132 228 228 228 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 136 136 136 241 241 241 F F F 20 400-419 131 131 131 225 225 225 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 137 137 137 242 242 242 F F F 24 480-499 137 137 137 242 242 242 F F F 25 500-519 136 136 136 241 241 241 F F F 26 520-539 136 136 136 241 241 241 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 136 136 136 240 240 240 F F F 29 580-599 134 134 134 232 232 232 F F F Max gen: 137, Max flush: 242 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.428374816Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.473282965Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.474029556Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.474081694Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.47409965Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.474122775Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736} {"msg":"Crucible 5a384d53-1a27-4bef-985c-fae1b343dfc6 has session id: 4b6ebd56-a449-4440-be6a-678c1a2f188e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.474728778Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Upstairs opts: Upstairs UUID: 5a384d53-1a27-4bef-985c-fae1b343dfc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.474769708Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Crucible stats registered with UUID: 5a384d53-1a27-4bef-985c-fae1b343dfc6","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.474787755Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:11.474803021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475281892Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475312226Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736} {"msg":"5a384d53-1a27-4bef-985c-fae1b343dfc6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475401525Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.47542783Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"0","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475461582Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"0","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475488916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"1","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475516899Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"1","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475542293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"2","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475568927Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"2","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475598961Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475622955Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.4756487Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52100)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475672674Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34785)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475696759Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40242)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.475774571Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.476380584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"2","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.476407588Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"0","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.477076086Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"2","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.477117777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"0","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.477415459Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"1","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.47767575Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.477702744Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.477951868Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"1","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[0]R flush_numbers[0..12]: [241, 238, 229, 236, 241, 223, 242, 242, 232, 241, 229, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478450095Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[0]R generation[0..12]: [136, 136, 133, 134, 136, 130, 137, 137, 134, 136, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478476839Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478491756Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[1]R flush_numbers[0..12]: [241, 238, 229, 236, 241, 223, 242, 242, 232, 241, 229, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478505793Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[1]R generation[0..12]: [136, 136, 133, 134, 136, 130, 137, 137, 134, 136, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478519419Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478533076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[2]R flush_numbers[0..12]: [241, 238, 229, 236, 241, 223, 242, 242, 232, 241, 229, 230]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478546603Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[2]R generation[0..12]: [136, 136, 133, 134, 136, 130, 137, 137, 134, 136, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.47856033Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478574037Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478586614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478599441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478611888Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478632514Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"":"downstairs","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478645391Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478657148Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"0","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478672395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"1","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478685362Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"client":"2","":"downstairs","session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478698269Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478710116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"5a384d53-1a27-4bef-985c-fae1b343dfc6 is now active with session: 4b6ebd56-a449-4440-be6a-678c1a2f188e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478722363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478734071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736,"session_id":"4b6ebd56-a449-4440-be6a-678c1a2f188e"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:11.478745908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 144 len 20480 data: 16 17 19 17 14 02/30 Write block 116 len 8192 data: 14 13 03/30 Write block 92 len 4096 data: 13 04/30 Write block 405 len 20480 data: 17 11 11 9 7 05/30 Read block 488 len 24576 06/30 Write block 388 len 20480 data: 14 14 15 12 12 07/30 Write block 233 len 32768 data: 16 16 15 13 13 12 10 11 08/30 Write block 76 len 36864 data: 7 8 7 9 10 15 16 16 15 09/30 Write block 396 len 36864 data: 8 8 11 10 11 13 17 18 15 10/30 Write block 123 len 8192 data: 8 7 11/30 Read block 235 len 36864 12/30 Read block 516 len 24576 13/30 Read block 127 len 4096 14/30 Write block 501 len 20480 data: 15 14 13 11 8 15/30 Write block 341 len 28672 data: 10 8 7 7 7 7 9 16/30 Read block 149 len 36864 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 262 len 16384 data: 10 12 13 13 21/30 Flush 22/30 Write block 173 len 36864 data: 13 13 12 13 16 14 12 13 14 23/30 Read block 320 len 16384 24/30 Write block 42 len 4096 data: 11 25/30 Flush 26/30 Write block 593 len 16384 data: 6 5 3 3 27/30 Write block 411 len 36864 data: 11 10 11 11 12 10 9 8 8 28/30 Write block 173 len 4096 data: 14 29/30 Read block 133 len 4096 30/30 Write block 594 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:11.496152406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1736} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 138 138 138 243 243 243 F F F 5 100-119 138 138 138 243 243 243 F F F 6 120-139 138 138 138 243 243 243 F F F 7 140-159 138 138 138 243 243 243 F F F 8 160-179 138 138 138 247 247 247 T T T 9 180-199 138 138 138 247 247 247 F F F 10 200-219 133 133 133 229 229 229 F F F 11 220-239 138 138 138 243 243 243 F F F 12 240-259 138 138 138 243 243 243 F F F 13 260-279 138 138 138 246 246 246 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 136 136 136 241 241 241 F F F 16 320-339 133 133 133 229 229 229 F F F 17 340-359 138 138 138 243 243 243 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 138 138 138 243 243 243 F F F 20 400-419 138 138 138 243 243 243 T T T 21 420-439 127 127 127 214 214 214 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 137 137 137 242 242 242 F F F 24 480-499 137 137 137 242 242 242 F F F 25 500-519 138 138 138 243 243 243 F F F 26 520-539 136 136 136 241 241 241 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 136 136 136 240 240 240 F F F 29 580-599 134 134 134 232 232 232 T T T Max gen: 138, Max flush: 247 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 138 138 138 243 243 243 F F F 5 100-119 138 138 138 243 243 243 F F F 6 120-139 138 138 138 243 243 243 F F F 7 140-159 138 138 138 243 243 243 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 138 138 138 247 247 247 F F F 10 200-219 133 133 133 229 229 229 F F F 11 220-239 138 138 138 243 243 243 F F F 12 240-259 138 138 138 243 243 243 F F F 13 260-279 138 138 138 246 246 246 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 136 136 136 241 241 241 F F F 16 320-339 133 133 133 229 229 229 F F F 17 340-359 138 138 138 243 243 243 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 138 138 138 243 243 243 F F F 20 400-419 139 139 139 248 248 248 F F F 21 420-439 127 127 127 214 214 214 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 137 137 137 242 242 242 F F F 24 480-499 137 137 137 242 242 242 F F F 25 500-519 138 138 138 243 243 243 F F F 26 520-539 136 136 136 241 241 241 F F F 27 540-559 129 129 129 220 220 220 F F F 28 560-579 136 136 136 240 240 240 F F F 29 580-599 139 139 139 248 248 248 F F F Max gen: 139, Max flush: 248 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.728318221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.773322828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.774335409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.774377009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.774391916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.774406282Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744} {"msg":"Crucible d2c0db3c-b1e9-4e50-9af2-09c163f54ab4 has session id: 6167032a-8e0e-4805-987a-4a0fa4886be2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.77512235Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Upstairs opts: Upstairs UUID: d2c0db3c-b1e9-4e50-9af2-09c163f54ab4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.77516988Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Crucible stats registered with UUID: d2c0db3c-b1e9-4e50-9af2-09c163f54ab4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775193744Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:15.775214399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775790139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775817653Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744} {"msg":"d2c0db3c-b1e9-4e50-9af2-09c163f54ab4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775908032Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775926368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"0","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775949283Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"0","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775967009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"1","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775981825Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"1","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.775994713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"2","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.77600845Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"2","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.776021596Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.776035104Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.77604898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50368)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.776186939Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"ds_connection connected from Ok(127.0.0.1:43261)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.776233219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55143)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.776265661Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.776908066Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"1","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.776946177Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"0","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.777603298Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"0","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.777630902Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"1","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.778187456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.77834493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.778799318Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"2","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779321039Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"2","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[0]R flush_numbers[0..12]: [241, 238, 247, 243, 243, 243, 243, 243, 248, 247, 229, 243]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.7798033Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[0]R generation[0..12]: [136, 136, 138, 138, 138, 138, 138, 138, 139, 138, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779827554Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779842511Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[1]R flush_numbers[0..12]: [241, 238, 247, 243, 243, 243, 243, 243, 248, 247, 229, 243]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779856908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[1]R generation[0..12]: [136, 136, 138, 138, 138, 138, 138, 138, 139, 138, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779870595Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779884392Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[2]R flush_numbers[0..12]: [241, 238, 247, 243, 243, 243, 243, 243, 248, 247, 229, 243]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779897919Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[2]R generation[0..12]: [136, 136, 138, 138, 138, 138, 138, 138, 139, 138, 133, 138]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779911216Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779924852Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779939599Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779956355Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779969502Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.779988768Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"":"downstairs","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.780007134Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.780027489Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"0","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.780042036Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"1","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.780055243Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"client":"2","":"downstairs","session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.78006832Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.780080547Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"d2c0db3c-b1e9-4e50-9af2-09c163f54ab4 is now active with session: 6167032a-8e0e-4805-987a-4a0fa4886be2","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.780093034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.780104672Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744,"session_id":"6167032a-8e0e-4805-987a-4a0fa4886be2"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:15.780123487Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 12288 data: 13 11 10 02/30 Write block 531 len 4096 data: 8 03/30 Write block 524 len 20480 data: 19 18 18 13 13 04/30 Read block 54 len 20480 05/30 Read block 409 len 8192 06/30 Read block 213 len 4096 07/30 Write block 418 len 36864 data: 9 9 10 8 7 9 9 10 10 08/30 Read block 126 len 12288 09/30 Write block 145 len 28672 data: 18 20 18 15 13 10 8 10/30 Read block 8 len 8192 11/30 Write block 482 len 12288 data: 11 9 9 12/30 Write block 426 len 20480 data: 11 10 9 10 8 13/30 Write block 148 len 4096 data: 16 14/30 Write block 555 len 24576 data: 5 5 6 6 6 8 15/30 Read block 410 len 32768 16/30 Flush 17/30 Write block 215 len 28672 data: 10 13 13 15 13 12 13 18/30 Write block 463 len 28672 data: 11 11 11 13 13 14 12 19/30 Read block 385 len 24576 20/30 Read block 304 len 4096 21/30 Read block 110 len 12288 22/30 Write block 196 len 4096 data: 11 23/30 Read block 35 len 16384 24/30 Write block 512 len 28672 data: 11 12 11 12 12 11 12 25/30 Read block 39 len 8192 26/30 Read block 97 len 32768 27/30 Write block 329 len 16384 data: 9 9 8 8 28/30 Read block 398 len 20480 29/30 Read block 82 len 32768 30/30 Read block 129 len 40960 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:15.798428402Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1744} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 138 138 138 243 243 243 F F F 5 100-119 138 138 138 243 243 243 F F F 6 120-139 138 138 138 243 243 243 F F F 7 140-159 140 140 140 249 249 249 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 140 140 140 249 249 249 T T T 10 200-219 133 133 133 229 229 229 T T T 11 220-239 138 138 138 243 243 243 T T T 12 240-259 138 138 138 243 243 243 F F F 13 260-279 138 138 138 246 246 246 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 136 136 136 241 241 241 F F F 16 320-339 133 133 133 229 229 229 T T T 17 340-359 138 138 138 243 243 243 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 138 138 138 243 243 243 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 137 137 137 242 242 242 T T T 24 480-499 140 140 140 249 249 249 F F F 25 500-519 138 138 138 243 243 243 T T T 26 520-539 140 140 140 249 249 249 F F F 27 540-559 140 140 140 249 249 249 F F F 28 560-579 140 140 140 249 249 249 F F F 29 580-599 139 139 139 248 248 248 F F F Max gen: 140, Max flush: 249 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 138 138 138 243 243 243 F F F 5 100-119 138 138 138 243 243 243 F F F 6 120-139 138 138 138 243 243 243 F F F 7 140-159 140 140 140 249 249 249 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 141 141 141 250 250 250 F F F 10 200-219 141 141 141 250 250 250 F F F 11 220-239 141 141 141 250 250 250 F F F 12 240-259 138 138 138 243 243 243 F F F 13 260-279 138 138 138 246 246 246 F F F 14 280-299 130 130 130 222 222 222 F F F 15 300-319 136 136 136 241 241 241 F F F 16 320-339 141 141 141 250 250 250 F F F 17 340-359 138 138 138 243 243 243 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 138 138 138 243 243 243 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 141 141 141 250 250 250 F F F 24 480-499 140 140 140 249 249 249 F F F 25 500-519 141 141 141 250 250 250 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 140 140 140 249 249 249 F F F 28 560-579 140 140 140 249 249 249 F F F 29 580-599 139 139 139 248 248 248 F F F Max gen: 141, Max flush: 250 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.043852093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.090392393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.091217996Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.091262186Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.091277512Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.091292749Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752} {"msg":"Crucible 9828b3e1-05a3-4241-ba82-a1f92cae0048 has session id: 9a30ff50-0942-4b35-9b80-8e19956311b4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.091919567Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Upstairs opts: Upstairs UUID: 9828b3e1-05a3-4241-ba82-a1f92cae0048, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.091953049Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Crucible stats registered with UUID: 9828b3e1-05a3-4241-ba82-a1f92cae0048","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.091974055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:20.091987192Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092384852Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092418804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752} {"msg":"9828b3e1-05a3-4241-ba82-a1f92cae0048 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092472412Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092502045Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"0","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092541876Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"0","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092572439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"1","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092634725Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"1","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092715527Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"2","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092752538Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"2","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092781942Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092816744Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092928058Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:47031)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.092984806Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:50359)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.093024837Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:48737)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.093047202Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.093549118Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"2","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.09358479Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"0","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.09371887Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"1","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.094307956Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"0","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.094349816Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"2","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.094381189Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"1","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.094869859Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.094987352Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[0]R flush_numbers[0..12]: [241, 238, 247, 243, 243, 243, 243, 249, 248, 250, 250, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095069204Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[0]R generation[0..12]: [136, 136, 138, 138, 138, 138, 138, 140, 139, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095090009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095104686Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[1]R flush_numbers[0..12]: [241, 238, 247, 243, 243, 243, 243, 249, 248, 250, 250, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095118393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[1]R generation[0..12]: [136, 136, 138, 138, 138, 138, 138, 140, 139, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095135709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095150045Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[2]R flush_numbers[0..12]: [241, 238, 247, 243, 243, 243, 243, 249, 248, 250, 250, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095163972Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[2]R generation[0..12]: [136, 136, 138, 138, 138, 138, 138, 140, 139, 141, 141, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095177709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095191546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095204763Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.09521817Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095231037Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095249993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"":"downstairs","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.09526293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095274667Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"0","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095287754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"1","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095300671Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"client":"2","":"downstairs","session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095313658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095325786Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"9828b3e1-05a3-4241-ba82-a1f92cae0048 is now active with session: 9a30ff50-0942-4b35-9b80-8e19956311b4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095340882Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095352869Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752,"session_id":"9a30ff50-0942-4b35-9b80-8e19956311b4"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:20.095365087Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 356 len 12288 data: 16 15 12 02/30 Write block 211 len 32768 data: 11 12 11 9 11 14 14 16 03/30 Read block 554 len 28672 04/30 Write block 512 len 4096 data: 12 05/30 Read block 60 len 40960 06/30 Write block 97 len 4096 data: 16 07/30 Read block 65 len 24576 08/30 Read block 397 len 20480 09/30 Write block 288 len 32768 data: 6 9 9 8 7 6 5 3 10/30 Read block 99 len 12288 11/30 Read block 327 len 36864 12/30 Write block 588 len 32768 data: 9 9 9 9 8 7 7 4 13/30 Write block 349 len 40960 data: 13 11 12 12 15 15 14 17 16 13 14/30 Read block 256 len 8192 15/30 Read block 86 len 4096 16/30 Write block 135 len 40960 data: 8 11 8 11 10 12 11 14 13 17 17/30 Read block 161 len 24576 18/30 Read block 324 len 40960 19/30 Read block 314 len 12288 20/30 Flush 21/30 Write block 484 len 36864 data: 10 9 12 14 14 13 10 8 7 22/30 Flush 23/30 Write block 334 len 28672 data: 12 15 13 11 11 12 11 24/30 Read block 242 len 12288 25/30 Read block 256 len 28672 26/30 Read block 396 len 8192 27/30 Write block 548 len 32768 data: 11 11 9 9 5 4 4 6 28/30 Read block 255 len 4096 29/30 Write block 276 len 28672 data: 5 6 10 10 11 13 12 30/30 Write block 281 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:20.115435741Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1752} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 142 142 142 251 251 251 F F F 5 100-119 138 138 138 243 243 243 F F F 6 120-139 142 142 142 251 251 251 F F F 7 140-159 142 142 142 251 251 251 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 141 141 141 250 250 250 F F F 10 200-219 142 142 142 251 251 251 F F F 11 220-239 141 141 141 250 250 250 F F F 12 240-259 138 138 138 243 243 243 F F F 13 260-279 138 138 138 246 246 246 T T T 14 280-299 142 142 142 251 251 251 T T T 15 300-319 136 136 136 241 241 241 F F F 16 320-339 141 141 141 250 250 250 T T T 17 340-359 142 142 142 251 251 251 T T T 18 360-379 134 134 134 230 230 230 F F F 19 380-399 138 138 138 243 243 243 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 141 141 141 250 250 250 F F F 24 480-499 142 142 142 252 252 252 F F F 25 500-519 142 142 142 251 251 251 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 140 140 140 249 249 249 T T T 28 560-579 140 140 140 249 249 249 F F F 29 580-599 142 142 142 251 251 251 F F F Max gen: 142, Max flush: 252 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 142 142 142 251 251 251 F F F 5 100-119 138 138 138 243 243 243 F F F 6 120-139 142 142 142 251 251 251 F F F 7 140-159 142 142 142 251 251 251 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 141 141 141 250 250 250 F F F 10 200-219 142 142 142 251 251 251 F F F 11 220-239 141 141 141 250 250 250 F F F 12 240-259 138 138 138 243 243 243 F F F 13 260-279 143 143 143 253 253 253 F F F 14 280-299 143 143 143 253 253 253 F F F 15 300-319 136 136 136 241 241 241 F F F 16 320-339 143 143 143 253 253 253 F F F 17 340-359 143 143 143 253 253 253 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 138 138 138 243 243 243 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 134 134 134 232 232 232 F F F 23 460-479 141 141 141 250 250 250 F F F 24 480-499 142 142 142 252 252 252 F F F 25 500-519 142 142 142 251 251 251 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 143 143 143 253 253 253 F F F 28 560-579 140 140 140 249 249 249 F F F 29 580-599 142 142 142 251 251 251 F F F Max gen: 143, Max flush: 253 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.359010941Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.404902158Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.405645009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.405703146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.405725201Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.405740248Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760} {"msg":"Crucible 09dd7279-688c-4ee1-8777-73cc51d8d609 has session id: 8d28a5a4-af24-4985-9127-bc98531e4a2c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.406339362Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Upstairs opts: Upstairs UUID: 09dd7279-688c-4ee1-8777-73cc51d8d609, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.406385512Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Crucible stats registered with UUID: 09dd7279-688c-4ee1-8777-73cc51d8d609","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.406410326Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:24.406430642Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.406854786Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.40687953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760} {"msg":"09dd7279-688c-4ee1-8777-73cc51d8d609 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.406975828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407006611Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"0","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407041633Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"0","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407064148Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"1","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407084403Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"1","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407108868Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"2","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407141031Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"2","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407167515Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407194229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407220263Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:41404)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407246877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61889)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407272851Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37548)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407433424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.40789666Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"1","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407934791Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"0","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.407965034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"2","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.408490365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"0","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.408582164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"1","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.408653918Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"2","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409039991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409127781Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[0]R flush_numbers[0..12]: [241, 238, 247, 243, 251, 243, 251, 251, 248, 250, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409236616Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[0]R generation[0..12]: [136, 136, 138, 138, 142, 138, 142, 142, 139, 141, 142, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409256352Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409270838Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[1]R flush_numbers[0..12]: [241, 238, 247, 243, 251, 243, 251, 251, 248, 250, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409284755Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[1]R generation[0..12]: [136, 136, 138, 138, 142, 138, 142, 142, 139, 141, 142, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409298392Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409311999Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[2]R flush_numbers[0..12]: [241, 238, 247, 243, 251, 243, 251, 251, 248, 250, 251, 250]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409325416Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[2]R generation[0..12]: [136, 136, 138, 138, 142, 138, 142, 142, 139, 141, 142, 141]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409338643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.4093522Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409364717Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409377394Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409389981Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409402509Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"":"downstairs","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409418335Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-01-15T05:24:24.409437241Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"0","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409450708Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"1","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409463445Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"client":"2","":"downstairs","session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409476382Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409488409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"09dd7279-688c-4ee1-8777-73cc51d8d609 is now active with session: 8d28a5a4-af24-4985-9127-bc98531e4a2c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409500796Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409512334Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760,"session_id":"8d28a5a4-af24-4985-9127-bc98531e4a2c"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:24.409524271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 274 len 8192 data: 7 6 02/30 Read block 471 len 24576 03/30 Read block 41 len 32768 04/30 Read block 180 len 40960 05/30 Read block 211 len 28672 06/30 Read block 403 len 36864 07/30 Write block 88 len 24576 data: 12 13 13 11 14 10 08/30 Write block 256 len 16384 data: 11 10 11 12 09/30 Read block 453 len 8192 10/30 Flush 11/30 Write block 213 len 20480 data: 12 10 12 15 15 12/30 Read block 240 len 16384 13/30 Read block 263 len 40960 14/30 Write block 589 len 16384 data: 10 10 10 9 15/30 Read block 175 len 36864 16/30 Read block 201 len 32768 17/30 Read block 4 len 24576 18/30 Flush 19/30 Write block 431 len 40960 data: 9 12 12 13 12 10 11 13 12 12 20/30 Read block 131 len 8192 21/30 Read block 225 len 28672 22/30 Write block 3 len 36864 data: 9 9 9 9 9 10 9 8 7 23/30 Write block 149 len 8192 data: 14 11 24/30 Read block 373 len 4096 25/30 Write block 308 len 36864 data: 10 10 9 9 8 8 9 11 13 26/30 Read block 244 len 16384 27/30 Write block 463 len 12288 data: 12 12 12 28/30 Read block 466 len 20480 29/30 Write block 221 len 20480 data: 14 17 22 19 20 30/30 Write block 494 len 36864 data: 9 9 9 10 12 13 15 16 15 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:24.430023238Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1760} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 241 241 241 T T T 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 144 144 144 254 254 254 F F F 5 100-119 138 138 138 243 243 243 F F F 6 120-139 142 142 142 251 251 251 F F F 7 140-159 142 142 142 251 251 251 T T T 8 160-179 139 139 139 248 248 248 F F F 9 180-199 141 141 141 250 250 250 F F F 10 200-219 144 144 144 255 255 255 F F F 11 220-239 141 141 141 250 250 250 T T T 12 240-259 144 144 144 254 254 254 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 143 143 143 253 253 253 F F F 15 300-319 136 136 136 241 241 241 T T T 16 320-339 143 143 143 253 253 253 F F F 17 340-359 143 143 143 253 253 253 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 138 138 138 243 243 243 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 T T T 22 440-459 134 134 134 232 232 232 T T T 23 460-479 141 141 141 250 250 250 T T T 24 480-499 142 142 142 252 252 252 T T T 25 500-519 142 142 142 251 251 251 T T T 26 520-539 140 140 140 249 249 249 F F F 27 540-559 143 143 143 253 253 253 F F F 28 560-579 140 140 140 249 249 249 F F F 29 580-599 144 144 144 255 255 255 F F F Max gen: 144, Max flush: 255 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 256 256 256 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 144 144 144 254 254 254 F F F 5 100-119 138 138 138 243 243 243 F F F 6 120-139 142 142 142 251 251 251 F F F 7 140-159 145 145 145 256 256 256 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 141 141 141 250 250 250 F F F 10 200-219 144 144 144 255 255 255 F F F 11 220-239 145 145 145 256 256 256 F F F 12 240-259 144 144 144 254 254 254 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 143 143 143 253 253 253 F F F 15 300-319 145 145 145 256 256 256 F F F 16 320-339 143 143 143 253 253 253 F F F 17 340-359 143 143 143 253 253 253 F F F 18 360-379 134 134 134 230 230 230 F F F 19 380-399 138 138 138 243 243 243 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 145 145 145 256 256 256 F F F 22 440-459 145 145 145 256 256 256 F F F 23 460-479 145 145 145 256 256 256 F F F 24 480-499 145 145 145 256 256 256 F F F 25 500-519 145 145 145 256 256 256 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 143 143 143 253 253 253 F F F 28 560-579 140 140 140 249 249 249 F F F 29 580-599 144 144 144 255 255 255 F F F Max gen: 145, Max flush: 256 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.698146208Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:28.742063122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.742526967Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.775083483Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.775828374Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.775864386Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.775878193Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.77589264Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768} {"msg":"Crucible fbb4a4fb-71de-456f-8d9e-e3746f7ce775 has session id: 14afcb1d-b1da-4be3-a44f-1eee50386c99","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.776497443Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Upstairs opts: Upstairs UUID: fbb4a4fb-71de-456f-8d9e-e3746f7ce775, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.77659772Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Crucible stats registered with UUID: fbb4a4fb-71de-456f-8d9e-e3746f7ce775","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.776622594Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:28.77664092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.77708501Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777109154Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768} {"msg":"fbb4a4fb-71de-456f-8d9e-e3746f7ce775 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777180368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777197974Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"0","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777221519Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"0","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777245673Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"1","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777272967Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"1","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777301281Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"2","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777331424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"2","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777359997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777390261Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777419754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"ds_connection connected from Ok(127.0.0.1:61195)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777448157Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"ds_connection connected from Ok(127.0.0.1:45610)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777474562Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"ds_connection connected from Ok(127.0.0.1:52120)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.777534948Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.778210485Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"1","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.778251195Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"2","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.778480534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"0","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.778976031Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"1","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779004875Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"2","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779021121Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"0","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779603959Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779628603Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[0]R flush_numbers[0..12]: [256, 238, 247, 243, 254, 243, 251, 256, 248, 250, 255, 256]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779653308Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[0]R generation[0..12]: [145, 136, 138, 138, 144, 138, 142, 145, 139, 141, 144, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779669434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779684081Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[1]R flush_numbers[0..12]: [256, 238, 247, 243, 254, 243, 251, 256, 248, 250, 255, 256]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779697828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[1]R generation[0..12]: [145, 136, 138, 138, 144, 138, 142, 145, 139, 141, 144, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779711365Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779725002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[2]R flush_numbers[0..12]: [256, 238, 247, 243, 254, 243, 251, 256, 248, 250, 255, 256]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779742508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[2]R generation[0..12]: [145, 136, 138, 138, 144, 138, 142, 145, 139, 141, 144, 145]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779769212Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779783009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779795506Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779808383Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.77982084Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779833407Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"":"downstairs","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779846194Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779858102Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"0","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779871229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"1","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Transition from WaitQuorum to Active"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779892364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"client":"2","":"downstairs","session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779905301Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779917218Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"fbb4a4fb-71de-456f-8d9e-e3746f7ce775 is now active with session: 14afcb1d-b1da-4be3-a44f-1eee50386c99","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779929675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.779941223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768,"session_id":"14afcb1d-b1da-4be3-a44f-1eee50386c99"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:28.77995614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 241 len 20480 data: 14 14 13 11 12 02/30 Flush 03/30 Write block 108 len 8192 data: 13 10 04/30 Write block 320 len 12288 data: 13 16 14 05/30 Read block 132 len 12288 06/30 Read block 103 len 40960 07/30 Read block 339 len 24576 08/30 Flush 09/30 Write block 377 len 24576 data: 15 13 13 13 11 11 10/30 Read block 2 len 40960 11/30 Write block 242 len 12288 data: 15 14 12 12/30 Write block 208 len 20480 data: 10 11 8 12 13 13/30 Write block 438 len 28672 data: 14 13 13 12 12 13 14 14/30 Flush 15/30 Write block 412 len 8192 data: 11 12 16/30 Flush 17/30 Write block 408 len 24576 data: 10 8 9 12 12 13 18/30 Read block 31 len 12288 19/30 Read block 259 len 36864 20/30 Read block 556 len 32768 21/30 Write block 410 len 40960 data: 10 13 13 14 12 13 11 10 10 10 22/30 Read block 395 len 24576 23/30 Write block 257 len 4096 data: 11 24/30 Read block 130 len 12288 25/30 Flush 26/30 Write block 474 len 12288 data: 11 11 12 27/30 Write block 193 len 20480 data: 11 14 12 12 10 28/30 Write block 103 len 24576 data: 15 12 13 11 12 14 29/30 Write block 232 len 24576 data: 16 17 17 16 14 14 30/30 Read block 170 len 20480 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:28.798699415Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1768} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 256 256 256 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 144 144 144 254 254 254 F F F 5 100-119 146 146 146 258 258 258 T T T 6 120-139 142 142 142 251 251 251 F F F 7 140-159 145 145 145 256 256 256 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 141 141 141 250 250 250 T T T 10 200-219 146 146 146 259 259 259 F F F 11 220-239 145 145 145 256 256 256 T T T 12 240-259 146 146 146 261 261 261 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 143 143 143 253 253 253 F F F 15 300-319 145 145 145 256 256 256 F F F 16 320-339 146 146 146 258 258 258 F F F 17 340-359 143 143 143 253 253 253 F F F 18 360-379 146 146 146 259 259 259 F F F 19 380-399 146 146 146 259 259 259 F F F 20 400-419 146 146 146 261 261 261 F F F 21 420-439 146 146 146 259 259 259 F F F 22 440-459 146 146 146 259 259 259 F F F 23 460-479 145 145 145 256 256 256 T T T 24 480-499 145 145 145 256 256 256 F F F 25 500-519 145 145 145 256 256 256 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 143 143 143 253 253 253 F F F 28 560-579 140 140 140 249 249 249 F F F 29 580-599 144 144 144 255 255 255 F F F Max gen: 146, Max flush: 261 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 256 256 256 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 144 144 144 254 254 254 F F F 5 100-119 147 147 147 262 262 262 F F F 6 120-139 142 142 142 251 251 251 F F F 7 140-159 145 145 145 256 256 256 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 147 147 147 262 262 262 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 147 147 147 262 262 262 F F F 12 240-259 146 146 146 261 261 261 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 143 143 143 253 253 253 F F F 15 300-319 145 145 145 256 256 256 F F F 16 320-339 146 146 146 258 258 258 F F F 17 340-359 143 143 143 253 253 253 F F F 18 360-379 146 146 146 259 259 259 F F F 19 380-399 146 146 146 259 259 259 F F F 20 400-419 146 146 146 261 261 261 F F F 21 420-439 146 146 146 259 259 259 F F F 22 440-459 146 146 146 259 259 259 F F F 23 460-479 147 147 147 262 262 262 F F F 24 480-499 145 145 145 256 256 256 F F F 25 500-519 145 145 145 256 256 256 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 143 143 143 253 253 253 F F F 28 560-579 140 140 140 249 249 249 F F F 29 580-599 144 144 144 255 255 255 F F F Max gen: 147, Max flush: 262 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.070121687Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.115385796Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.116133306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.116176867Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.116202211Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.116230335Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776} {"msg":"Crucible ff4228d6-fa21-4396-9e8b-2451a4593402 has session id: 780f6b3b-4530-4577-8b39-fce272991407","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.116794197Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Upstairs opts: Upstairs UUID: ff4228d6-fa21-4396-9e8b-2451a4593402, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.116835447Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Crucible stats registered with UUID: ff4228d6-fa21-4396-9e8b-2451a4593402","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.116854243Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:33.11686767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117389952Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117416046Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776} {"msg":"ff4228d6-fa21-4396-9e8b-2451a4593402 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117476212Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117494368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"0","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117516913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"0","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.11753073Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"1","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117549916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"1","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.11757418Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"2","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117598005Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"2","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117623599Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117649623Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117686155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"ds_connection connected from Ok(127.0.0.1:46704)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117716648Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"ds_connection connected from Ok(127.0.0.1:46795)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117746761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"ds_connection connected from Ok(127.0.0.1:65348)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.117774135Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.118490712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"1","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.118520916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"0","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.11863553Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"2","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119174248Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"1","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119202721Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"0","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119284293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"2","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119731451Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119819512Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[0]R flush_numbers[0..12]: [256, 238, 247, 243, 254, 262, 251, 256, 248, 262, 259, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119929127Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[0]R generation[0..12]: [145, 136, 138, 138, 144, 147, 142, 145, 139, 147, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119949052Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119968518Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[1]R flush_numbers[0..12]: [256, 238, 247, 243, 254, 262, 251, 256, 248, 262, 259, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119983114Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[1]R generation[0..12]: [145, 136, 138, 138, 144, 147, 142, 145, 139, 147, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.119996791Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120010738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[2]R flush_numbers[0..12]: [256, 238, 247, 243, 254, 262, 251, 256, 248, 262, 259, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120024375Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[2]R generation[0..12]: [145, 136, 138, 138, 144, 147, 142, 145, 139, 147, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120037742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120051389Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120064196Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120077143Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.12009011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120102907Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"":"downstairs","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120115804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:24:33.120127682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"0","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120152666Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"1","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120167693Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"client":"2","":"downstairs","session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.1201809Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120193117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"ff4228d6-fa21-4396-9e8b-2451a4593402 is now active with session: 780f6b3b-4530-4577-8b39-fce272991407","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120205614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120217221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776,"session_id":"780f6b3b-4530-4577-8b39-fce272991407"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:33.120229149Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 303 len 4096 data: 10 02/30 Read block 501 len 12288 03/30 Read block 489 len 32768 04/30 Read block 554 len 8192 05/30 Read block 340 len 24576 06/30 Flush 07/30 Write block 146 len 4096 data: 21 08/30 Write block 111 len 8192 data: 8 8 09/30 Write block 82 len 40960 data: 17 17 16 12 12 12 13 14 14 12 10/30 Read block 87 len 40960 11/30 Read block 223 len 20480 12/30 Read block 61 len 12288 13/30 Read block 65 len 8192 14/30 Flush 15/30 Write block 549 len 36864 data: 12 10 10 6 5 5 7 6 7 16/30 Read block 378 len 40960 17/30 Write block 394 len 4096 data: 7 18/30 Write block 118 len 4096 data: 12 19/30 Write block 574 len 8192 data: 11 12 20/30 Write block 133 len 40960 data: 10 8 9 12 9 12 11 13 12 15 21/30 Read block 409 len 20480 22/30 Read block 269 len 4096 23/30 Write block 408 len 40960 data: 11 9 11 14 14 15 13 14 12 11 24/30 Read block 143 len 36864 25/30 Read block 54 len 8192 26/30 Write block 189 len 36864 data: 15 15 15 15 12 15 13 13 11 27/30 Read block 229 len 16384 28/30 Read block 569 len 12288 29/30 Write block 593 len 8192 data: 8 8 30/30 Read block 168 len 28672 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 9 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 16 14 14 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:33.139150424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1776} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 256 256 256 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 148 148 148 264 264 264 F F F 5 100-119 148 148 148 264 264 264 T T T 6 120-139 142 142 142 251 251 251 T T T 7 140-159 148 148 148 264 264 264 T T T 8 160-179 139 139 139 248 248 248 F F F 9 180-199 147 147 147 262 262 262 T T T 10 200-219 146 146 146 259 259 259 F F F 11 220-239 147 147 147 262 262 262 F F F 12 240-259 146 146 146 261 261 261 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 143 143 143 253 253 253 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 146 146 146 258 258 258 F F F 17 340-359 143 143 143 253 253 253 F F F 18 360-379 146 146 146 259 259 259 F F F 19 380-399 146 146 146 259 259 259 T T T 20 400-419 146 146 146 261 261 261 T T T 21 420-439 146 146 146 259 259 259 F F F 22 440-459 146 146 146 259 259 259 F F F 23 460-479 147 147 147 262 262 262 F F F 24 480-499 145 145 145 256 256 256 F F F 25 500-519 145 145 145 256 256 256 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 143 143 143 253 253 253 T T T 28 560-579 140 140 140 249 249 249 T T T 29 580-599 144 144 144 255 255 255 T T T Max gen: 148, Max flush: 264 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 145 145 145 256 256 256 F F F 1 020-039 136 136 136 238 238 238 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 148 148 148 264 264 264 F F F 5 100-119 149 149 149 265 265 265 F F F 6 120-139 149 149 149 265 265 265 F F F 7 140-159 149 149 149 265 265 265 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 149 149 149 265 265 265 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 147 147 147 262 262 262 F F F 12 240-259 146 146 146 261 261 261 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 143 143 143 253 253 253 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 146 146 146 258 258 258 F F F 17 340-359 143 143 143 253 253 253 F F F 18 360-379 146 146 146 259 259 259 F F F 19 380-399 149 149 149 265 265 265 F F F 20 400-419 149 149 149 265 265 265 F F F 21 420-439 146 146 146 259 259 259 F F F 22 440-459 146 146 146 259 259 259 F F F 23 460-479 147 147 147 262 262 262 F F F 24 480-499 145 145 145 256 256 256 F F F 25 500-519 145 145 145 256 256 256 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 149 149 149 265 265 265 F F F 28 560-579 149 149 149 265 265 265 F F F 29 580-599 149 149 149 265 265 265 F F F Max gen: 149, Max flush: 265 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.402762116Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.448913353Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.44976573Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.449804522Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.449818858Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.449832945Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784} {"msg":"Crucible cdb7efde-c13c-49db-8091-9eb615828630 has session id: 39a43aa1-a0b2-4d06-bfee-af25de261943","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.450458583Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Upstairs opts: Upstairs UUID: cdb7efde-c13c-49db-8091-9eb615828630, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.450572928Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Crucible stats registered with UUID: cdb7efde-c13c-49db-8091-9eb615828630","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.450593683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:37.45060661Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451076234Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451094439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784} {"msg":"cdb7efde-c13c-49db-8091-9eb615828630 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451172512Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451189008Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"0","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451216002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"0","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451230229Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"1","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451244136Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"1","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451258102Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"2","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451271989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"2","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451284726Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451306522Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451331606Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"ds_connection connected from Ok(127.0.0.1:59012)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451391792Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"ds_connection connected from Ok(127.0.0.1:56866)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451424315Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"ds_connection connected from Ok(127.0.0.1:63400)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.451601735Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.452166856Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"1","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.452206378Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"0","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.452320932Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"2","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.452898671Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"1","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.452943141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"0","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.452963806Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"2","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453509332Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453578057Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[0]R flush_numbers[0..12]: [256, 238, 247, 243, 264, 265, 265, 265, 248, 265, 259, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453640103Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[0]R generation[0..12]: [145, 136, 138, 138, 148, 149, 149, 149, 139, 149, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453660608Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453676245Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[1]R flush_numbers[0..12]: [256, 238, 247, 243, 264, 265, 265, 265, 248, 265, 259, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453690371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[1]R generation[0..12]: [145, 136, 138, 138, 148, 149, 149, 149, 139, 149, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453703888Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453717385Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[2]R flush_numbers[0..12]: [256, 238, 247, 243, 264, 265, 265, 265, 248, 265, 259, 262]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453730742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[2]R generation[0..12]: [145, 136, 138, 138, 148, 149, 149, 149, 139, 149, 146, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453744119Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453757676Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453775052Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453788619Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453801356Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:24:37.453813923Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"":"downstairs","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453833729Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453845626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"0","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453858693Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"1","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.45387135Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"client":"2","":"downstairs","session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453884398Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453896515Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"cdb7efde-c13c-49db-8091-9eb615828630 is now active with session: 39a43aa1-a0b2-4d06-bfee-af25de261943","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453909122Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453920829Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784,"session_id":"39a43aa1-a0b2-4d06-bfee-af25de261943"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:37.453932916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 335 len 24576 data: 16 14 12 12 13 12 03/30 Read block 491 len 40960 04/30 Read block 128 len 40960 05/30 Write block 505 len 4096 data: 9 06/30 Read block 168 len 28672 07/30 Write block 281 len 28672 data: 15 13 9 8 7 7 8 08/30 Read block 119 len 8192 09/30 Write block 379 len 8192 data: 14 14 10/30 Read block 37 len 4096 11/30 Write block 237 len 4096 data: 15 12/30 Read block 579 len 16384 13/30 Read block 190 len 32768 14/30 Read block 255 len 16384 15/30 Read block 391 len 16384 16/30 Write block 155 len 20480 data: 10 13 18 16 17 17/30 Write block 288 len 24576 data: 7 10 10 9 8 7 18/30 Flush 19/30 Write block 8 len 28672 data: 11 10 9 8 9 7 7 20/30 Write block 25 len 20480 data: 8 11 10 11 10 21/30 Flush 22/30 Write block 444 len 16384 data: 15 13 11 13 23/30 Read block 338 len 8192 24/30 Read block 497 len 40960 25/30 Write block 256 len 20480 data: 12 12 12 13 12 26/30 Read block 370 len 4096 27/30 Write block 194 len 12288 data: 16 14 14 28/30 Write block 413 len 20480 data: 16 14 15 13 12 29/30 Write block 459 len 8192 data: 9 7 30/30 Write block 190 len 28672 data: 16 16 16 13 17 15 15 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 8 7 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:37.47184324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1784} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 148 148 148 264 264 264 F F F 5 100-119 149 149 149 265 265 265 F F F 6 120-139 149 149 149 265 265 265 F F F 7 140-159 150 150 150 267 267 267 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 149 149 149 265 265 265 T T T 10 200-219 146 146 146 259 259 259 F F F 11 220-239 150 150 150 267 267 267 F F F 12 240-259 146 146 146 261 261 261 T T T 13 260-279 144 144 144 254 254 254 T T T 14 280-299 150 150 150 267 267 267 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 150 150 150 267 267 267 F F F 17 340-359 150 150 150 267 267 267 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 150 150 150 267 267 267 F F F 20 400-419 149 149 149 265 265 265 T T T 21 420-439 146 146 146 259 259 259 F F F 22 440-459 146 146 146 259 259 259 T T T 23 460-479 147 147 147 262 262 262 T T T 24 480-499 145 145 145 256 256 256 F F F 25 500-519 150 150 150 267 267 267 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 149 149 149 265 265 265 F F F 28 560-579 149 149 149 265 265 265 F F F 29 580-599 149 149 149 265 265 265 F F F Max gen: 150, Max flush: 268 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 148 148 148 264 264 264 F F F 5 100-119 149 149 149 265 265 265 F F F 6 120-139 149 149 149 265 265 265 F F F 7 140-159 150 150 150 267 267 267 F F F 8 160-179 139 139 139 248 248 248 F F F 9 180-199 151 151 151 269 269 269 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 150 150 150 267 267 267 F F F 12 240-259 151 151 151 269 269 269 F F F 13 260-279 151 151 151 269 269 269 F F F 14 280-299 150 150 150 267 267 267 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 150 150 150 267 267 267 F F F 17 340-359 150 150 150 267 267 267 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 150 150 150 267 267 267 F F F 20 400-419 151 151 151 269 269 269 F F F 21 420-439 146 146 146 259 259 259 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 151 151 151 269 269 269 F F F 24 480-499 145 145 145 256 256 256 F F F 25 500-519 150 150 150 267 267 267 F F F 26 520-539 140 140 140 249 249 249 F F F 27 540-559 149 149 149 265 265 265 F F F 28 560-579 149 149 149 265 265 265 F F F 29 580-599 149 149 149 265 265 265 F F F Max gen: 151, Max flush: 269 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.718472927Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:41.76314216Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.763531232Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.796319536Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.796960361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.797004351Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.797021847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.797039253Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792} {"msg":"Crucible 7df17314-09d4-47ac-97dd-83cd525ba953 has session id: 913f2739-9f6b-4b2b-9d7b-f5462e233338","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.799716827Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Upstairs opts: Upstairs UUID: 7df17314-09d4-47ac-97dd-83cd525ba953, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.799756418Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Crucible stats registered with UUID: 7df17314-09d4-47ac-97dd-83cd525ba953","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.799772734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:41.799784892Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800195328Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800215654Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792} {"msg":"7df17314-09d4-47ac-97dd-83cd525ba953 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800289577Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800307453Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"0","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800330468Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"0","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800344025Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"1","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800357962Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"1","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800370919Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"2","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800385145Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"2","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800397983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.80041125Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800426526Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"ds_connection connected from Ok(127.0.0.1:46527)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800455649Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"ds_connection connected from Ok(127.0.0.1:46395)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800691546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"ds_connection connected from Ok(127.0.0.1:40528)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.800728628Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.801145553Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"1","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.801175406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"2","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.801678273Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"0","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.801716754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"2","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.801879567Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"1","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802232707Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"0","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802275857Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.80235066Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[0]R flush_numbers[0..12]: [268, 268, 247, 243, 264, 265, 265, 267, 248, 269, 259, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802741662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[0]R generation[0..12]: [150, 150, 138, 138, 148, 149, 149, 150, 139, 151, 146, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802762097Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802776724Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[1]R flush_numbers[0..12]: [268, 268, 247, 243, 264, 265, 265, 267, 248, 269, 259, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.80279461Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[1]R generation[0..12]: [150, 150, 138, 138, 148, 149, 149, 150, 139, 151, 146, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802808677Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802822564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[2]R flush_numbers[0..12]: [268, 268, 247, 243, 264, 265, 265, 267, 248, 269, 259, 267]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.80283622Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[2]R generation[0..12]: [150, 150, 138, 138, 148, 149, 149, 150, 139, 151, 146, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802849807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802863414Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802876131Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802889098Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802902006Z",Wait for a query_work_queue command to finish before sending IO "hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802921401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"":"downstairs","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802934198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802945925Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"0","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802958943Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"1","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.80297169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"client":"2","":"downstairs","session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802984547Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.802998964Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"7df17314-09d4-47ac-97dd-83cd525ba953 is now active with session: 913f2739-9f6b-4b2b-9d7b-f5462e233338","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.803019619Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.803032956Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792,"session_id":"913f2739-9f6b-4b2b-9d7b-f5462e233338"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:41.803045163Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 140 len 24576 data: 14 13 16 14 18 19 02/30 Read block 277 len 4096 03/30 Read block 462 len 28672 04/30 Read block 358 len 12288 05/30 Write block 175 len 40960 data: 13 14 17 15 13 14 15 13 13 11 06/30 Read block 540 len 28672 07/30 Write block 589 len 12288 data: 11 11 11 08/30 Flush 09/30 Write block 253 len 32768 data: 9 9 8 13 13 13 14 13 10/30 Write block 221 len 16384 data: 15 18 23 20 11/30 Read block 230 len 28672 12/30 Read block 351 len 36864 13/30 Write block 269 len 12288 data: 9 7 6 14/30 Read block 379 len 12288 15/30 Read block 329 len 20480 16/30 Read block 417 len 16384 17/30 Flush 18/30 Write block 531 len 4096 data: 9 19/30 Read block 278 len 32768 20/30 Write block 224 len 32768 data: 21 21 17 19 18 19 18 18 21/30 Read block 104 len 12288 22/30 Read block 82 len 28672 23/30 Write block 576 len 4096 data: 13 24/30 Read block 236 len 4096 25/30 Write block 150 len 8192 data: 12 9 26/30 Read block 310 len 12288 27/30 Write block 564 len 16384 data: 7 4 6 8 28/30 Read block 140 len 40960 29/30 Read block 367 len 32768 30/30 Write block 328 len 8192 data: 9 10 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 1 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:41.822322827Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1792} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 148 148 148 264 264 264 F F F 5 100-119 149 149 149 265 265 265 F F F 6 120-139 149 149 149 265 265 265 F F F 7 140-159 152 152 152 270 270 270 T T T 8 160-179 152 152 152 270 270 270 F F F 9 180-199 152 152 152 270 270 270 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 152 152 152 271 271 271 T T T 12 240-259 152 152 152 271 271 271 F F F 13 260-279 152 152 152 271 271 271 F F F 14 280-299 150 150 150 267 267 267 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 150 150 150 267 267 267 T T T 17 340-359 150 150 150 267 267 267 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 150 150 150 267 267 267 F F F 20 400-419 151 151 151 269 269 269 F F F 21 420-439 146 146 146 259 259 259 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 151 151 151 269 269 269 F F F 24 480-499 145 145 145 256 256 256 F F F 25 500-519 150 150 150 267 267 267 F F F 26 520-539 140 140 140 249 249 249 T T T 27 540-559 149 149 149 265 265 265 F F F 28 560-579 149 149 149 265 265 265 T T T 29 580-599 152 152 152 270 270 270 F F F Max gen: 152, Max flush: 271 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 F F F 4 080-099 148 148 148 264 264 264 F F F 5 100-119 149 149 149 265 265 265 F F F 6 120-139 149 149 149 265 265 265 F F F 7 140-159 153 153 153 272 272 272 F F F 8 160-179 152 152 152 270 270 270 F F F 9 180-199 152 152 152 270 270 270 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 153 153 153 272 272 272 F F F 12 240-259 152 152 152 271 271 271 F F F 13 260-279 152 152 152 271 271 271 F F F 14 280-299 150 150 150 267 267 267 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 153 153 153 272 272 272 F F F 17 340-359 150 150 150 267 267 267 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 150 150 150 267 267 267 F F F 20 400-419 151 151 151 269 269 269 F F F 21 420-439 146 146 146 259 259 259 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 151 151 151 269 269 269 F F F 24 480-499 145 145 145 256 256 256 F F F 25 500-519 150 150 150 267 267 267 F F F 26 520-539 153 153 153 272 272 272 F F F 27 540-559 149 149 149 265 265 265 F F F 28 560-579 153 153 153 272 272 272 F F F 29 580-599 152 152 152 270 270 270 F F F Max gen: 153, Max flush: 272 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.10090001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:46.144308629Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.14469865Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.17675741Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.177482305Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.177516698Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.177531214Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.177545751Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800} {"msg":"Crucible 1ad68ab4-d3d6-4325-8214-4555e8b92c28 has session id: 86421689-2288-4cf1-86a8-1ff014836fca","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178233075Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Upstairs opts: Upstairs UUID: 1ad68ab4-d3d6-4325-8214-4555e8b92c28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178278055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Crucible stats registered with UUID: 1ad68ab4-d3d6-4325-8214-4555e8b92c28","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.17830112Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:46.178314787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178669966Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178702739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800} {"msg":"1ad68ab4-d3d6-4325-8214-4555e8b92c28 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178758476Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178784731Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"0","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178812764Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"0","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178827821Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"1","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178842267Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"1","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178855115Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"2","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178868841Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"2","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178881778Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178895115Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.17891959Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"ds_connection connected from Ok(127.0.0.1:60900)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178946324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"ds_connection connected from Ok(127.0.0.1:49601)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.178973038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"ds_connection connected from Ok(127.0.0.1:39930)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.179000802Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.17975901Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"2","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.179795322Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"0","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.179823275Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"1","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.180425329Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"1","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.180531665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"0","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.180562248Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"2","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.180986022Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181116992Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[0]R flush_numbers[0..12]: [268, 268, 247, 243, 264, 265, 265, 272, 270, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181141187Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[0]R generation[0..12]: [150, 150, 138, 138, 148, 149, 149, 153, 152, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181156093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.18117025Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[1]R flush_numbers[0..12]: [268, 268, 247, 243, 264, 265, 265, 272, 270, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181184157Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[1]R generation[0..12]: [150, 150, 138, 138, 148, 149, 149, 153, 152, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181197574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181211291Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[2]R flush_numbers[0..12]: [268, 268, 247, 243, 264, 265, 265, 272, 270, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181224598Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[2]R generation[0..12]: [150, 150, 138, 138, 148, 149, 149, 153, 152, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181237975Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181251432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181267568Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181280765Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181293492Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181306019Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"":"downstairs","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181318997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Transition from WaitQuorum to Active","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181339532Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"0","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181352579Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"1","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181365506Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"client":"2","":"downstairs","session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181379333Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.1813913Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"1ad68ab4-d3d6-4325-8214-4555e8b92c28 is now active with session: 86421689-2288-4cf1-86a8-1ff014836fca","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181404137Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181415804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800,"session_id":"86421689-2288-4cf1-86a8-1ff014836fca"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:46.181427662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 266 len 36864 data: 14 13 14 10 8 7 6 7 8 02/30 Write block 103 len 20480 data: 16 13 14 12 13 03/30 Read block 305 len 24576 04/30 Read block 581 len 4096 05/30 Write block 125 len 8192 data: 6 7 06/30 Read block 290 len 16384 07/30 Read block 186 len 12288 08/30 Write block 240 len 20480 data: 12 15 16 15 13 09/30 Write block 426 len 28672 data: 12 11 10 11 9 10 13 10/30 Read block 197 len 28672 11/30 Read block 233 len 8192 12/30 Read block 210 len 16384 13/30 Write block 421 len 20480 data: 9 8 10 10 11 14/30 Flush 15/30 Write block 78 len 28672 data: 8 10 11 16 18 18 17 16/30 Write block 345 len 12288 data: 8 8 10 17/30 Read block 528 len 24576 18/30 Read block 53 len 28672 19/30 Write block 267 len 16384 data: 14 15 11 9 20/30 Read block 509 len 32768 21/30 Read block 112 len 40960 22/30 Read block 401 len 24576 23/30 Read block 236 len 32768 24/30 Read block 334 len 28672 25/30 Read block 363 len 16384 26/30 Write block 478 len 16384 data: 7 12 12 13 27/30 Read block 568 len 24576 28/30 Write block 551 len 32768 data: 11 7 6 6 8 7 8 7 29/30 Read block 474 len 12288 30/30 Read block 364 len 40960 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 7 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:46.201812425Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1800} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 138 138 138 243 243 243 T T T 4 080-099 148 148 148 264 264 264 T T T 5 100-119 154 154 154 273 273 273 F F F 6 120-139 154 154 154 273 273 273 F F F 7 140-159 153 153 153 272 272 272 F F F 8 160-179 152 152 152 270 270 270 F F F 9 180-199 152 152 152 270 270 270 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 153 153 153 272 272 272 F F F 12 240-259 154 154 154 273 273 273 F F F 13 260-279 154 154 154 273 273 273 T T T 14 280-299 150 150 150 267 267 267 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 153 153 153 272 272 272 F F F 17 340-359 150 150 150 267 267 267 T T T 18 360-379 150 150 150 267 267 267 F F F 19 380-399 150 150 150 267 267 267 F F F 20 400-419 151 151 151 269 269 269 F F F 21 420-439 154 154 154 273 273 273 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 151 151 151 269 269 269 T T T 24 480-499 145 145 145 256 256 256 T T T 25 500-519 150 150 150 267 267 267 F F F 26 520-539 153 153 153 272 272 272 F F F 27 540-559 149 149 149 265 265 265 T T T 28 560-579 153 153 153 272 272 272 F F F 29 580-599 152 152 152 270 270 270 F F F Max gen: 154, Max flush: 273 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 138 138 138 247 247 247 F F F 3 060-079 155 155 155 274 274 274 F F F 4 080-099 155 155 155 274 274 274 F F F 5 100-119 154 154 154 273 273 273 F F F 6 120-139 154 154 154 273 273 273 F F F 7 140-159 153 153 153 272 272 272 F F F 8 160-179 152 152 152 270 270 270 F F F 9 180-199 152 152 152 270 270 270 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 153 153 153 272 272 272 F F F 12 240-259 154 154 154 273 273 273 F F F 13 260-279 155 155 155 274 274 274 F F F 14 280-299 150 150 150 267 267 267 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 153 153 153 272 272 272 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 150 150 150 267 267 267 F F F 20 400-419 151 151 151 269 269 269 F F F 21 420-439 154 154 154 273 273 273 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 155 155 155 274 274 274 F F F 24 480-499 155 155 155 274 274 274 F F F 25 500-519 150 150 150 267 267 267 F F F 26 520-539 153 153 153 272 272 272 F F F 27 540-559 155 155 155 274 274 274 F F F 28 560-579 153 153 153 272 272 272 F F F 29 580-599 152 152 152 270 270 270 F F F Max gen: 155, Max flush: 274 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.484241976Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.530812388Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.531524997Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.531566917Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.531582434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.53159715Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808} {"msg":"Crucible 374724bf-97b3-4e77-9344-dcbe1542ca65 has session id: 8e5cd63d-0f6a-4d68-8141-d0f7719e254f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532138168Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Upstairs opts: Upstairs UUID: 374724bf-97b3-4e77-9344-dcbe1542ca65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532177729Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Crucible stats registered with UUID: 374724bf-97b3-4e77-9344-dcbe1542ca65","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532198754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:50.532212171Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.53265951Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532684994Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808} {"msg":"374724bf-97b3-4e77-9344-dcbe1542ca65 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.53274573Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532770675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"0","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532795489Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"0","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532809366Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"1","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532823713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"1","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532838089Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"2","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532852176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"2","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532865553Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532890078Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532919361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46403)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532950294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56832)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.532979877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58396)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.533225632Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.533637359Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"1","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.533817658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"0","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.533888532Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"2","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.534368003Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"1","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.534395077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"0","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.534682142Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"2","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.534960509Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535038821Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[0]R flush_numbers[0..12]: [268, 268, 247, 274, 274, 273, 273, 272, 270, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535301032Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[0]R generation[0..12]: [150, 150, 138, 155, 155, 154, 154, 153, 152, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535322077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535336983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[1]R flush_numbers[0..12]: [268, 268, 247, 274, 274, 273, 273, 272, 270, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535355969Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[1]R generation[0..12]: [150, 150, 138, 155, 155, 154, 154, 153, 152, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535370166Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535384063Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[2]R flush_numbers[0..12]: [268, 268, 247, 274, 274, 273, 273, 272, 270, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.53539761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[2]R generation[0..12]: [150, 150, 138, 155, 155, 154, 154, 153, 152, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535411237Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535425094Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535438011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535451797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535464395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:24:50.535477062Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"":"downstairs","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535493058Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535504736Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"0","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535517643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"1","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.53553028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"client":"2","":"downstairs","session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535543307Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535557624Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"374724bf-97b3-4e77-9344-dcbe1542ca65 is now active with session: 8e5cd63d-0f6a-4d68-8141-d0f7719e254f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535570601Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535582408Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808,"session_id":"8e5cd63d-0f6a-4d68-8141-d0f7719e254f"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:24:50.535594235Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 436 len 4096 data: 11 02/30 Read block 113 len 8192 03/30 Read block 28 len 24576 04/30 Read block 67 len 24576 05/30 Read block 187 len 36864 06/30 Write block 406 len 20480 data: 12 12 12 10 12 07/30 Read block 0 len 16384 08/30 Write block 505 len 8192 data: 10 9 09/30 Write block 286 len 40960 data: 8 9 8 11 11 10 9 8 6 4 10/30 Write block 52 len 28672 data: 10 11 12 10 10 8 8 11/30 Read block 442 len 4096 12/30 Read block 58 len 12288 13/30 Read block 426 len 32768 14/30 Read block 7 len 12288 15/30 Write block 143 len 8192 data: 15 19 16/30 Read block 208 len 4096 17/30 Read block 99 len 16384 18/30 Write block 38 len 28672 data: 12 11 12 13 12 15 15 19/30 Read block 288 len 12288 20/30 Read block 231 len 32768 21/30 Write block 163 len 24576 data: 17 13 13 12 12 10 22/30 Write block 488 len 4096 data: 15 23/30 Write block 111 len 40960 data: 9 9 9 9 12 15 14 13 11 11 24/30 Write block 565 len 16384 data: 5 7 9 11 25/30 Read block 577 len 24576 26/30 Write block 261 len 36864 data: 13 11 13 14 14 15 15 16 12 27/30 Read block 406 len 16384 28/30 Read block 553 len 40960 29/30 Read block 169 len 4096 30/30 Read block 532 len 16384 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 5 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:24:50.555439861Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1808} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 150 150 150 268 268 268 T T T 2 040-059 138 138 138 247 247 247 T T T 3 060-079 155 155 155 274 274 274 F F F 4 080-099 155 155 155 274 274 274 F F F 5 100-119 154 154 154 273 273 273 T T T 6 120-139 154 154 154 273 273 273 T T T 7 140-159 153 153 153 272 272 272 T T T 8 160-179 152 152 152 270 270 270 T T T 9 180-199 152 152 152 270 270 270 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 153 153 153 272 272 272 F F F 12 240-259 154 154 154 273 273 273 F F F 13 260-279 155 155 155 274 274 274 T T T 14 280-299 150 150 150 267 267 267 T T T 15 300-319 148 148 148 263 263 263 F F F 16 320-339 153 153 153 272 272 272 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 150 150 150 267 267 267 F F F 20 400-419 151 151 151 269 269 269 T T T 21 420-439 154 154 154 273 273 273 T T T 22 440-459 151 151 151 269 269 269 F F F 23 460-479 155 155 155 274 274 274 F F F 24 480-499 155 155 155 274 274 274 T T T 25 500-519 150 150 150 267 267 267 T T T 26 520-539 153 153 153 272 272 272 F F F 27 540-559 155 155 155 274 274 274 F F F 28 560-579 153 153 153 272 272 272 T T T 29 580-599 152 152 152 270 270 270 F F F Max gen: 155, Max flush: 274 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 156 156 156 275 275 275 F F F 2 040-059 156 156 156 275 275 275 F F F 3 060-079 155 155 155 274 274 274 F F F 4 080-099 155 155 155 274 274 274 F F F 5 100-119 156 156 156 275 275 275 F F F 6 120-139 156 156 156 275 275 275 F F F 7 140-159 156 156 156 275 275 275 F F F 8 160-179 156 156 156 275 275 275 F F F 9 180-199 152 152 152 270 270 270 F F F 10 200-219 146 146 146 259 259 259 F F F 11 220-239 153 153 153 272 272 272 F F F 12 240-259 154 154 154 273 273 273 F F F 13 260-279 156 156 156 275 275 275 F F F 14 280-299 156 156 156 275 275 275 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 153 153 153 272 272 272 F F F 17 340-359 155 155 155 274 274 274 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 150 150 150 267 267 267 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 156 156 156 275 275 275 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 155 155 155 274 274 274 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 156 156 156 275 275 275 F F F 26 520-539 153 153 153 272 272 272 F F F 27 540-559 155 155 155 274 274 274 F F F 28 560-579 156 156 156 275 275 275 F F F 29 580-599 152 152 152 270 270 270 F F F Max gen: 156, Max flush: 275 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.845774328Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:04.889351948Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.889762745Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.922158898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.922825127Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.922866788Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.922883614Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.92290149Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816} {"msg":"Crucible 0a84b411-067d-4282-9073-eddc7fe99e15 has session id: d40ce460-5805-4a12-ade2-ca9913b5658a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.923534207Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Upstairs opts: Upstairs UUID: 0a84b411-067d-4282-9073-eddc7fe99e15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.923568159Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Crucible stats registered with UUID: 0a84b411-067d-4282-9073-eddc7fe99e15","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.923582976Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:04.923595563Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924029055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924061917Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816} {"msg":"0a84b411-067d-4282-9073-eddc7fe99e15 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924089491Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924106077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"0","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924134571Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"0","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924151887Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"1","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924169423Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"1","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924195747Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"2","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.92422746Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"2","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924256153Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924284877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.92431265Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38165)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924336405Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53037)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.92436009Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58289)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.924385234Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.92515302Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"2","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.925183933Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"0","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.92520006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"1","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.925779258Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"2","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.925876586Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"1","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.925896892Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"0","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.92630121Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926424972Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[0]R flush_numbers[0..12]: [268, 275, 275, 274, 274, 275, 275, 275, 275, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926542456Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[0]R generation[0..12]: [150, 156, 156, 155, 155, 156, 156, 156, 156, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926561181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926575448Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[1]R flush_numbers[0..12]: [268, 275, 275, 274, 274, 275, 275, 275, 275, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926589175Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[1]R generation[0..12]: [150, 156, 156, 155, 155, 156, 156, 156, 156, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926602632Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926616649Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[2]R flush_numbers[0..12]: [268, 275, 275, 274, 274, 275, 275, 275, 275, 270, 259, 272]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926629916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[2]R generation[0..12]: [150, 156, 156, 155, 155, 156, 156, 156, 156, 152, 146, 153]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926665348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926680304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Max found gen is 157","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926693421Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Generation requested: 158 >= found:157","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926706318Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Next flush: 276"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926724754Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926737671Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"":"downstairs","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926750508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926762176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"0","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926774963Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"1","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.9267879Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"client":"2","":"downstairs","session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926800747Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926812734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"0a84b411-067d-4282-9073-eddc7fe99e15 is now active with session: d40ce460-5805-4a12-ade2-ca9913b5658a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926825061Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926836739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816,"session_id":"d40ce460-5805-4a12-ade2-ca9913b5658a"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:04.926848556Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 398 len 4096 data: 12 02/30 Write block 33 len 32768 data: 10 9 8 8 9 13 12 13 03/30 Read block 232 len 32768 04/30 Write block 245 len 16384 data: 13 13 11 12 05/30 Read block 368 len 4096 06/30 Flush 07/30 Write block 188 len 4096 data: 14 08/30 Read block 217 len 36864 09/30 Write block 336 len 32768 data: 15 13 13 14 13 11 9 8 10/30 Read block 297 len 4096 11/30 Write block 394 len 8192 data: 8 7 12/30 Flush 13/30 Write block 256 len 24576 data: 14 14 14 15 14 14 14/30 Read block 504 len 32768 15/30 Write block 454 len 24576 data: 10 8 7 7 8 10 16/30 Read block 509 len 24576 17/30 Read block 322 len 40960 18/30 Write block 106 len 16384 data: 13 14 15 11 19/30 Read block 12 len 36864 20/30 Read block 386 len 24576 21/30 Read block 370 len 12288 22/30 Read block 462 len 40960 23/30 Read block 159 len 24576 24/30 Write block 216 len 16384 data: 16 16 17 14 25/30 Write block 231 len 28672 data: 19 17 18 18 17 15 16 26/30 Read block 27 len 20480 27/30 Read block 295 len 4096 28/30 Read block 522 len 8192 29/30 Write block 559 len 28672 data: 7 9 6 8 9 8 6 30/30 Write block 521 len 8192 data: 15 18 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 6 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:04.946289403Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1816} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 158 158 158 276 276 276 F F F 2 040-059 158 158 158 276 276 276 F F F 3 060-079 155 155 155 274 274 274 F F F 4 080-099 155 155 155 274 274 274 F F F 5 100-119 156 156 156 275 275 275 T T T 6 120-139 156 156 156 275 275 275 F F F 7 140-159 156 156 156 275 275 275 F F F 8 160-179 156 156 156 275 275 275 F F F 9 180-199 158 158 158 277 277 277 F F F 10 200-219 146 146 146 259 259 259 T T T 11 220-239 153 153 153 272 272 272 T T T 12 240-259 158 158 158 276 276 276 T T T 13 260-279 156 156 156 275 275 275 T T T 14 280-299 156 156 156 275 275 275 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 158 158 158 277 277 277 F F F 17 340-359 158 158 158 277 277 277 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 158 158 158 277 277 277 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 156 156 156 275 275 275 F F F 22 440-459 151 151 151 269 269 269 T T T 23 460-479 155 155 155 274 274 274 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 156 156 156 275 275 275 F F F 26 520-539 153 153 153 272 272 272 T T T 27 540-559 155 155 155 274 274 274 T T T 28 560-579 156 156 156 275 275 275 T T T 29 580-599 152 152 152 270 270 270 F F F Max gen: 158, Max flush: 277 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 F F F 1 020-039 158 158 158 276 276 276 F F F 2 040-059 158 158 158 276 276 276 F F F 3 060-079 155 155 155 274 274 274 F F F 4 080-099 155 155 155 274 274 274 F F F 5 100-119 159 159 159 278 278 278 F F F 6 120-139 156 156 156 275 275 275 F F F 7 140-159 156 156 156 275 275 275 F F F 8 160-179 156 156 156 275 275 275 F F F 9 180-199 158 158 158 277 277 277 F F F 10 200-219 159 159 159 278 278 278 F F F 11 220-239 159 159 159 278 278 278 F F F 12 240-259 159 159 159 278 278 278 F F F 13 260-279 159 159 159 278 278 278 F F F 14 280-299 156 156 156 275 275 275 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 158 158 158 277 277 277 F F F 17 340-359 158 158 158 277 277 277 F F F 18 360-379 150 150 150 267 267 267 F F F 19 380-399 158 158 158 277 277 277 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 156 156 156 275 275 275 F F F 22 440-459 159 159 159 278 278 278 F F F 23 460-479 155 155 155 274 274 274 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 156 156 156 275 275 275 F F F 26 520-539 159 159 159 278 278 278 F F F 27 540-559 159 159 159 278 278 278 F F F 28 560-579 159 159 159 278 278 278 F F F 29 580-599 152 152 152 270 270 270 F F F Max gen: 159, Max flush: 278 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.244483433Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:09.289178681Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.289685996Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.322550752Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.323158545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.323194877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.323212762Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.323227279Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824} {"msg":"Crucible 9a23fde5-9e10-462e-a480-5331c802032c has session id: e20383e6-cd62-4083-bed1-e984895b1d6f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.323848968Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Upstairs opts: Upstairs UUID: 9a23fde5-9e10-462e-a480-5331c802032c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.32388786Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Crucible stats registered with UUID: 9a23fde5-9e10-462e-a480-5331c802032c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.323904346Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:09.323917143Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324317352Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324341877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824} {"msg":"9a23fde5-9e10-462e-a480-5331c802032c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324396154Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324423828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"0","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324445973Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"0","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.32446029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"1","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324474797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"1","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324496492Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"2","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324523956Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"2","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.32454764Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324571715Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324596429Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62950)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324621993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42442)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324766291Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:43898)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.324888513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.325428191Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"1","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.325464112Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"2","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.325601421Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"0","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326097909Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"1","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326126273Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"2","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326145628Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"0","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326646765Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326734995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[0]R flush_numbers[0..12]: [268, 276, 276, 274, 274, 278, 275, 275, 275, 277, 278, 278]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326830503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[0]R generation[0..12]: [150, 158, 158, 155, 155, 159, 156, 156, 156, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326851089Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326865995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[1]R flush_numbers[0..12]: [268, 276, 276, 274, 274, 278, 275, 275, 275, 277, 278, 278]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326879972Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[1]R generation[0..12]: [150, 158, 158, 155, 155, 159, 156, 156, 156, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326893449Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326907296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[2]R flush_numbers[0..12]: [268, 276, 276, 274, 274, 278, 275, 275, 275, 277, 278, 278]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326920983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[2]R generation[0..12]: [150, 158, 158, 155, 155, 159, 156, 156, 156, 158, 159, 159]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326934339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326947887Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326960504Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326973381Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326986028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.326998585Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"":"downstairs","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.327015931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.327028408Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"0","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.327041635Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"client":"1","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.327054612Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid"Wait for a query_work_queue command to finish before sending IO :1824,"client":"2","":"downstairs","session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.327075887Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.327087855Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"9a23fde5-9e10-462e-a480-5331c802032c is now active with session: e20383e6-cd62-4083-bed1-e984895b1d6f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.3271086Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.327123387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824,"session_id":"e20383e6-cd62-4083-bed1-e984895b1d6f"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:09.327135644Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 59 len 28672 data: 8 8 11 10 12 13 15 02/30 Write block 424 len 12288 data: 11 12 13 03/30 Read block 558 len 4096 04/30 Write block 364 len 40960 data: 15 15 11 10 11 9 10 9 7 11 05/30 Write block 86 len 32768 data: 13 13 14 15 15 13 15 11 06/30 Write block 580 len 36864 data: 15 17 17 17 15 15 13 12 10 07/30 Read block 329 len 20480 08/30 Read block 583 len 16384 09/30 Read block 593 len 12288 10/30 Flush 11/30 Write block 429 len 16384 data: 12 10 11 14 12/30 Write block 239 len 20480 data: 11 13 16 17 16 13/30 Read block 481 len 28672 14/30 Write block 378 len 36864 data: 14 15 15 12 12 11 12 11 12 15/30 Read block 139 len 16384 16/30 Read block 359 len 36864 17/30 Read block 252 len 20480 18/30 Flush 19/30 Write block 8 len 4096 data: 12 20/30 Write block 551 len 28672 data: 12 8 7 7 9 8 9 21/30 Write block 543 len 40960 data: 12 13 13 12 12 12 13 11 13 9 22/30 Read block 171 len 20480 23/30 Flush 24/30 Write block 117 len 32768 data: 15 14 12 12 8 8 9 8 25/30 Write block 531 len 4096 data: 10 26/30 Read block 17 len 36864 27/30 Read block 404 len 36864 28/30 Read block 118 len 40960 29/30 Write block 290 len 20480 data: 12 11 10 9 7 30/30 Write block 43 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:09.346902087Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1824} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 158 158 158 276 276 276 F F F 2 040-059 160 160 160 279 279 279 T T T 3 060-079 160 160 160 279 279 279 F F F 4 080-099 160 160 160 279 279 279 F F F 5 100-119 159 159 159 278 278 278 T T T 6 120-139 156 156 156 275 275 275 T T T 7 140-159 156 156 156 275 275 275 F F F 8 160-179 156 156 156 275 275 275 F F F 9 180-199 158 158 158 277 277 277 F F F 10 200-219 159 159 159 278 278 278 F F F 11 220-239 160 160 160 280 280 280 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 159 159 159 278 278 278 F F F 14 280-299 156 156 156 275 275 275 T T T 15 300-319 148 148 148 263 263 263 F F F 16 320-339 158 158 158 277 277 277 F F F 17 340-359 158 158 158 277 277 277 F F F 18 360-379 160 160 160 280 280 280 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 160 160 160 280 280 280 F F F 22 440-459 159 159 159 278 278 278 F F F 23 460-479 155 155 155 274 274 274 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 156 156 156 275 275 275 F F F 26 520-539 159 159 159 278 278 278 T T T 27 540-559 160 160 160 281 281 281 F F F 28 560-579 159 159 159 278 278 278 F F F 29 580-599 160 160 160 279 279 279 F F F Max gen: 160, Max flush: 281 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 158 158 158 276 276 276 F F F 2 040-059 161 161 161 282 282 282 F F F 3 060-079 160 160 160 279 279 279 F F F 4 080-099 160 160 160 279 279 279 F F F 5 100-119 161 161 161 282 282 282 F F F 6 120-139 161 161 161 282 282 282 F F F 7 140-159 156 156 156 275 275 275 F F F 8 160-179 156 156 156 275 275 275 F F F 9 180-199 158 158 158 277 277 277 F F F 10 200-219 159 159 159 278 278 278 F F F 11 220-239 160 160 160 280 280 280 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 159 159 159 278 278 278 F F F 14 280-299 161 161 161 282 282 282 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 158 158 158 277 277 277 F F F 17 340-359 158 158 158 277 277 277 F F F 18 360-379 160 160 160 280 280 280 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 160 160 160 280 280 280 F F F 22 440-459 159 159 159 278 278 278 F F F 23 460-479 155 155 155 274 274 274 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 156 156 156 275 275 275 F F F 26 520-539 161 161 161 282 282 282 F F F 27 540-559 160 160 160 281 281 281 F F F 28 560-579 159 159 159 278 278 278 F F F 29 580-599 160 160 160 279 279 279 F F F Max gen: 161, Max flush: 282 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.622588705Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.669161107Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.6699408Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.66998433Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.670000057Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.670014653Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832} {"msg":"Crucible 217c44bb-dbfc-431e-ac19-892e654390fc has session id: 050030ed-c60a-4809-8f5a-035fbf43ac7e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.670660557Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Upstairs opts: Upstairs UUID: 217c44bb-dbfc-431e-ac19-892e654390fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.670707056Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Crucible stats registered with UUID: 217c44bb-dbfc-431e-ac19-892e654390fc","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.670732061Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:13.670755386Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671200835Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671226619Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832} {"msg":"217c44bb-dbfc-431e-ac19-892e654390fc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.67131119Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671329246Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"0","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.67135238Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"0","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671373856Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"1","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.67140027Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"1","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671428703Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"2","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671458536Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"2","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.67148587Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671515124Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671543647Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54490)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671570091Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61559)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671598665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52456)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.671830032Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.672331759Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"0","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.672358962Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"1","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.672381557Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"2","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.672970844Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"0","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673046447Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"1","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673241602Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"2","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673570098Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673589534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[0]R flush_numbers[0..12]: [281, 276, 282, 279, 279, 282, 282, 275, 275, 277, 278, 280]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673785629Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[0]R generation[0..12]: [160, 158, 161, 160, 160, 161, 161, 156, 156, 158, 159, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673804545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673818692Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[1]R flush_numbers[0..12]: [281, 276, 282, 279, 279, 282, 282, 275, 275, 277, 278, 280]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673832329Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[1]R generation[0..12]: [160, 158, 161, 160, 160, 161, 161, 156, 156, 158, 159, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673845746Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673859622Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[2]R flush_numbers[0..12]: [281, 276, 282, 279, 279, 282, 282, 275, 275, 277, 278, 280]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.67387321Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[2]R generation[0..12]: [160, 158, 161, 160, 160, 161, 161, 156, 156, 158, 159, 160]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673890846Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673904752Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.67391738Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673930577Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673943443Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673956061ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"":"downstairs","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673976126Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.673987944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"0","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.674016467Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"1","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.674030764Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"client":"2","":"downstairs","session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.674043761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.674055728Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"217c44bb-dbfc-431e-ac19-892e654390fc is now active with session: 050030ed-c60a-4809-8f5a-035fbf43ac7e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.674068355Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.674079983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832,"session_id":"050030ed-c60a-4809-8f5a-035fbf43ac7e"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:13.67409199Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 162 len 4096 data: 16 03/30 Read block 34 len 40960 04/30 Write block 73 len 24576 data: 9 9 7 8 9 9 05/30 Read block 394 len 12288 06/30 Flush 07/30 Write block 223 len 16384 data: 24 22 22 18 08/30 Read block 572 len 36864 09/30 Flush 10/30 Write block 509 len 28672 data: 9 9 9 13 13 12 13 11/30 Write block 198 len 40960 data: 11 12 11 10 13 10 8 8 8 10 12/30 Write block 458 len 20480 data: 9 11 8 9 10 13/30 Read block 449 len 12288 14/30 Write block 135 len 40960 data: 10 13 10 13 12 15 14 17 16 20 15/30 Read block 423 len 32768 16/30 Read block 374 len 40960 17/30 Write block 572 len 4096 data: 16 18/30 Read block 431 len 20480 19/30 Read block 362 len 4096 20/30 Read block 463 len 32768 21/30 Read block 380 len 24576 22/30 Read block 36 len 16384 23/30 Write block 440 len 28672 data: 14 13 13 14 16 14 12 24/30 Read block 561 len 4096 25/30 Read block 246 len 32768 26/30 Read block 325 len 16384 27/30 Write block 347 len 4096 data: 11 28/30 Read block 99 len 4096 29/30 Read block 332 len 36864 30/30 Read block 595 len 16384 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 7 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:13.694604934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1832} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 158 158 158 276 276 276 F F F 2 040-059 161 161 161 282 282 282 F F F 3 060-079 162 162 162 284 284 284 F F F 4 080-099 160 160 160 279 279 279 F F F 5 100-119 161 161 161 282 282 282 F F F 6 120-139 161 161 161 282 282 282 T T T 7 140-159 156 156 156 275 275 275 T T T 8 160-179 162 162 162 284 284 284 F F F 9 180-199 158 158 158 277 277 277 T T T 10 200-219 159 159 159 278 278 278 T T T 11 220-239 162 162 162 285 285 285 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 159 159 159 278 278 278 F F F 14 280-299 161 161 161 282 282 282 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 158 158 158 277 277 277 F F F 17 340-359 158 158 158 277 277 277 T T T 18 360-379 160 160 160 280 280 280 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 160 160 160 280 280 280 F F F 22 440-459 159 159 159 278 278 278 T T T 23 460-479 155 155 155 274 274 274 T T T 24 480-499 156 156 156 275 275 275 F F F 25 500-519 156 156 156 275 275 275 T T T 26 520-539 161 161 161 282 282 282 F F F 27 540-559 160 160 160 281 281 281 F F F 28 560-579 159 159 159 278 278 278 T T T 29 580-599 160 160 160 279 279 279 F F F Max gen: 162, Max flush: 285 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 158 158 158 276 276 276 F F F 2 040-059 161 161 161 282 282 282 F F F 3 060-079 162 162 162 284 284 284 F F F 4 080-099 160 160 160 279 279 279 F F F 5 100-119 161 161 161 282 282 282 F F F 6 120-139 163 163 163 286 286 286 F F F 7 140-159 163 163 163 286 286 286 F F F 8 160-179 162 162 162 284 284 284 F F F 9 180-199 163 163 163 286 286 286 F F F 10 200-219 163 163 163 286 286 286 F F F 11 220-239 162 162 162 285 285 285 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 159 159 159 278 278 278 F F F 14 280-299 161 161 161 282 282 282 F F F 15 300-319 148 148 148 263 263 263 F F F 16 320-339 158 158 158 277 277 277 F F F 17 340-359 163 163 163 286 286 286 F F F 18 360-379 160 160 160 280 280 280 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 F F F 21 420-439 160 160 160 280 280 280 F F F 22 440-459 163 163 163 286 286 286 F F F 23 460-479 163 163 163 286 286 286 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 163 163 163 286 286 286 F F F 26 520-539 161 161 161 282 282 282 F F F 27 540-559 160 160 160 281 281 281 F F F 28 560-579 163 163 163 286 286 286 F F F 29 580-599 160 160 160 279 279 279 F F F Max gen: 163, Max flush: 286 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:27.957994065Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:28.001488354Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.001932363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.03400195Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.03462163Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.03466464Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.034687545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.034706371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840} {"msg":"Crucible d8eb3d32-6972-4475-a327-7fbed71271ca has session id: 756c2e06-e0e6-4779-9141-759dded07dba","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035270153Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Upstairs opts: Upstairs UUID: d8eb3d32-6972-4475-a327-7fbed71271ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035320641Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Crucible stats registered with UUID: d8eb3d32-6972-4475-a327-7fbed71271ca","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035337958Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:28.035350555Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035686739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035712593Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840} {"msg":"d8eb3d32-6972-4475-a327-7fbed71271ca active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035775309Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035792415Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"0","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035816269Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"0","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035830336Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"1","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035844373Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"1","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.03585745Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"2","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035877406Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"2","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035891212Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035904889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.035918286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"ds_connection connected from Ok(127.0.0.1:33096)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.036048257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"ds_connection connected from Ok(127.0.0.1:59421)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.036268477Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"ds_connection connected from Ok(127.0.0.1:36059)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.036324404Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.036978366Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"2","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.037008399Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"1","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.037024526Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"0","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.037635847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"1","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.037665561Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"2","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.037681797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"0","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038218405Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038247499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[0]R flush_numbers[0..12]: [281, 276, 282, 284, 279, 282, 286, 286, 284, 286, 286, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038269853Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[0]R generation[0..12]: [160, 158, 161, 162, 160, 161, 163, 163, 162, 163, 163, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.03828465Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038299697Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[1]R flush_numbers[0..12]: [281, 276, 282, 284, 279, 282, 286, 286, 284, 286, 286, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038313584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[1]R generation[0..12]: [160, 158, 161, 162, 160, 161, 163, 163, 162, 163, 163, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038326911Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038340557Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[2]R flush_numbers[0..12]: [281, 276, 282, 284, 279, 282, 286, 286, 284, 286, 286, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038353954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[2]R generation[0..12]: [160, 158, 161, 162, 160, 161, 163, 163, 162, 163, 163, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038367502Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038381048Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038393565Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038406483Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.03841915Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038431767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"":"downstairs","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038446564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038458351Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"0","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038479446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"1","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038492683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"client":"2","":"downstairs","session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.03850573Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038517788Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"d8eb3d32-6972-4475-a327-7fbed71271ca is now active with session: 756c2e06-e0e6-4779-9141-759dded07dba","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038530235Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038541872Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840,"session_id":"756c2e06-e0e6-4779-9141-759dded07dba"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:28.038553599Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 579 len 20480 data: 17 16 18 18 18 02/30 Write block 321 len 32768 data: 17 15 13 11 8 8 8 10 03/30 Write block 278 len 8192 data: 11 11 04/30 Read block 305 len 4096 05/30 Write block 362 len 28672 data: 12 12 16 16 12 11 12 06/30 Write block 22 len 40960 data: 8 7 6 9 12 11 12 11 11 11 07/30 Read block 151 len 12288 08/30 Read block 35 len 4096 09/30 Write block 157 len 28672 data: 19 17 18 17 16 17 18 10/30 Write block 94 len 16384 data: 12 14 15 17 11/30 Write block 344 len 32768 data: 8 9 9 12 11 14 12 13 12/30 Write block 542 len 4096 data: 10 13/30 Read block 329 len 32768 14/30 Read block 551 len 28672 15/30 Write block 175 len 20480 data: 14 15 18 16 14 16/30 Flush 17/30 Write block 306 len 32768 data: 9 10 11 11 10 10 9 9 18/30 Write block 417 len 40960 data: 13 11 11 11 10 9 11 12 13 14 19/30 Read block 84 len 40960 20/30 Write block 169 len 8192 data: 12 10 21/30 Read block 287 len 36864 22/30 Read block 489 len 40960 23/30 Read block 100 len 12288 24/30 Write block 356 len 12288 data: 18 17 14 25/30 Write block 27 len 8192 data: 12 13 26/30 Read block 100 len 20480 27/30 Read block 493 len 32768 28/30 Write block 295 len 28672 data: 5 2 2 2 2 2 5 29/30 Write block 110 len 24576 data: 8 10 10 10 10 13 30/30 Write block 166 len 32768 data: 13 13 11 13 11 11 12 15 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:28.057148818Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1840} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 164 164 164 287 287 287 T T T 2 040-059 161 161 161 282 282 282 F F F 3 060-079 162 162 162 284 284 284 F F F 4 080-099 164 164 164 287 287 287 F F F 5 100-119 161 161 161 282 282 282 T T T 6 120-139 163 163 163 286 286 286 F F F 7 140-159 164 164 164 287 287 287 F F F 8 160-179 164 164 164 287 287 287 T T T 9 180-199 163 163 163 286 286 286 F F F 10 200-219 163 163 163 286 286 286 F F F 11 220-239 162 162 162 285 285 285 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 164 164 164 287 287 287 F F F 14 280-299 161 161 161 282 282 282 T T T 15 300-319 148 148 148 263 263 263 T T T 16 320-339 164 164 164 287 287 287 F F F 17 340-359 164 164 164 287 287 287 T T T 18 360-379 164 164 164 287 287 287 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 156 156 156 275 275 275 T T T 21 420-439 160 160 160 280 280 280 T T T 22 440-459 163 163 163 286 286 286 F F F 23 460-479 163 163 163 286 286 286 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 163 163 163 286 286 286 F F F 26 520-539 161 161 161 282 282 282 F F F 27 540-559 164 164 164 287 287 287 F F F 28 560-579 164 164 164 287 287 287 F F F 29 580-599 164 164 164 287 287 287 F F F Max gen: 164, Max flush: 287 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 165 165 165 288 288 288 F F F 2 040-059 161 161 161 282 282 282 F F F 3 060-079 162 162 162 284 284 284 F F F 4 080-099 164 164 164 287 287 287 F F F 5 100-119 165 165 165 288 288 288 F F F 6 120-139 163 163 163 286 286 286 F F F 7 140-159 164 164 164 287 287 287 F F F 8 160-179 165 165 165 288 288 288 F F F 9 180-199 163 163 163 286 286 286 F F F 10 200-219 163 163 163 286 286 286 F F F 11 220-239 162 162 162 285 285 285 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 164 164 164 287 287 287 F F F 14 280-299 165 165 165 288 288 288 F F F 15 300-319 165 165 165 288 288 288 F F F 16 320-339 164 164 164 287 287 287 F F F 17 340-359 165 165 165 288 288 288 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 165 165 165 288 288 288 F F F 21 420-439 165 165 165 288 288 288 F F F 22 440-459 163 163 163 286 286 286 F F F 23 460-479 163 163 163 286 286 286 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 163 163 163 286 286 286 F F F 26 520-539 161 161 161 282 282 282 F F F 27 540-559 164 164 164 287 287 287 F F F 28 560-579 164 164 164 287 287 287 F F F 29 580-599 164 164 164 287 287 287 F F F Max gen: 165, Max flush: 288 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.344481198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.390432111Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.391103099Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.391146889Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.391162905Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.391177222Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848} {"msg":"Crucible 1b0d5492-afc7-4d94-8f07-bdf5616f9705 has session id: 408f8668-df99-4eea-8e91-513d3a76f01f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.391792113Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Upstairs opts: Upstairs UUID: 1b0d5492-afc7-4d94-8f07-bdf5616f9705, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.391838073Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Crucible stats registered with UUID: 1b0d5492-afc7-4d94-8f07-bdf5616f9705","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.391862517Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:32.391882562Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392385019Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392410793Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848} {"msg":"1b0d5492-afc7-4d94-8f07-bdf5616f9705 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392479787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392497753Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"0","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392522857Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"0","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392546092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"1","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392572576Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"1","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.39259947Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"2","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392628773Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"2","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392653508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392678762Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392703177Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34917)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392729891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54004)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392755705Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64553)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.392781219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.393451907Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"0","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.393479601Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"2","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.393495387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"1","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394091432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"1","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394119626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"0","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.39418919Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"2","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394682648Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394799452Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[0]R flush_numbers[0..12]: [281, 288, 282, 284, 287, 288, 286, 287, 288, 286, 286, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394867577Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[0]R generation[0..12]: [160, 165, 161, 162, 164, 165, 163, 164, 165, 163, 163, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394887702Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394902518Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[1]R flush_numbers[0..12]: [281, 288, 282, 284, 287, 288, 286, 287, 288, 286, 286, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394916415Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[1]R generation[0..12]: [160, 165, 161, 162, 164, 165, 163, 164, 165, 163, 163, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394929922Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394943589Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[2]R flush_numbers[0..12]: [281, 288, 282, 284, 287, 288, 286, 287, 288, 286, 286, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394960995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[2]R generation[0..12]: [160, 165, 161, 162, 164, 165, 163, 164, 165, 163, 163, 162]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394974792Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.394988379Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395001076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395013943Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.39502667Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395039308Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"":"downstairs","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395052275Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395069401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"0","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395082518Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"1","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395095315Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"client":"2","":"downstairs","session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395108272Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395120249Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"1b0d5492-afc7-4d94-8f07-bdf5616f9705 is now active with session: 408f8668-df99-4eea-8e91-513d3a76f01f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395132846Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395144504Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848,"session_id":"408f8668-df99-4eea-8e91-513d3a76f01f"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:32.395158541Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 139 len 20480 data: 13 16 15 18 17 02/30 Write block 568 len 36864 data: 12 13 13 15 17 14 12 13 14 03/30 Read block 80 len 16384 04/30 Read block 403 len 36864 05/30 Read block 379 len 32768 06/30 Write block 278 len 16384 data: 12 12 12 16 07/30 Flush 08/30 Write block 90 len 8192 data: 16 14 09/30 Write block 427 len 40960 data: 12 11 13 11 12 15 13 14 13 12 10/30 Write block 222 len 28672 data: 19 25 23 23 19 20 19 11/30 Read block 221 len 28672 12/30 Read block 61 len 12288 13/30 Flush 14/30 Write block 319 len 24576 data: 12 14 18 16 14 12 15/30 Read block 223 len 32768 16/30 Write block 410 len 24576 data: 13 15 15 17 15 16 17/30 Read block 573 len 24576 18/30 Write block 336 len 12288 data: 16 14 14 19/30 Read block 572 len 20480 20/30 Write block 152 len 16384 data: 5 6 7 11 21/30 Read block 542 len 8192 22/30 Flush 23/30 Write block 314 len 36864 data: 10 12 14 12 11 13 15 19 17 24/30 Write block 526 len 32768 data: 19 14 14 13 10 11 8 9 25/30 Read block 178 len 24576 26/30 Write block 452 len 40960 data: 11 9 11 9 8 8 10 12 9 10 27/30 Write block 545 len 32768 data: 14 13 13 13 14 12 14 10 28/30 Read block 550 len 12288 29/30 Read block 215 len 36864 30/30 Read block 113 len 32768 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 8 7 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:32.415332802Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1848} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 165 165 165 288 288 288 F F F 2 040-059 161 161 161 282 282 282 F F F 3 060-079 162 162 162 284 284 284 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 165 165 165 288 288 288 F F F 6 120-139 166 166 166 289 289 289 F F F 7 140-159 166 166 166 291 291 291 F F F 8 160-179 165 165 165 288 288 288 F F F 9 180-199 163 163 163 286 286 286 F F F 10 200-219 163 163 163 286 286 286 F F F 11 220-239 166 166 166 290 290 290 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 166 166 166 289 289 289 F F F 14 280-299 166 166 166 289 289 289 F F F 15 300-319 166 166 166 291 291 291 T T T 16 320-339 166 166 166 291 291 291 T T T 17 340-359 165 165 165 288 288 288 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 166 166 166 291 291 291 F F F 21 420-439 166 166 166 290 290 290 F F F 22 440-459 163 163 163 286 286 286 T T T 23 460-479 163 163 163 286 286 286 T T T 24 480-499 156 156 156 275 275 275 F F F 25 500-519 163 163 163 286 286 286 F F F 26 520-539 161 161 161 282 282 282 T T T 27 540-559 164 164 164 287 287 287 T T T 28 560-579 166 166 166 289 289 289 F F F 29 580-599 164 164 164 287 287 287 F F F Max gen: 166, Max flush: 291 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 281 281 281 F F F 1 020-039 165 165 165 288 288 288 F F F 2 040-059 161 161 161 282 282 282 F F F 3 060-079 162 162 162 284 284 284 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 165 165 165 288 288 288 F F F 6 120-139 166 166 166 289 289 289 F F F 7 140-159 166 166 166 291 291 291 F F F 8 160-179 165 165 165 288 288 288 F F F 9 180-199 163 163 163 286 286 286 F F F 10 200-219 163 163 163 286 286 286 F F F 11 220-239 166 166 166 290 290 290 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 166 166 166 289 289 289 F F F 14 280-299 166 166 166 289 289 289 F F F 15 300-319 167 167 167 292 292 292 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 165 165 165 288 288 288 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 160 160 160 280 280 280 F F F 20 400-419 166 166 166 291 291 291 F F F 21 420-439 166 166 166 290 290 290 F F F 22 440-459 167 167 167 292 292 292 F F F 23 460-479 167 167 167 292 292 292 F F F 24 480-499 156 156 156 275 275 275 F F F 25 500-519 163 163 163 286 286 286 F F F 26 520-539 167 167 167 292 292 292 F F F 27 540-559 167 167 167 292 292 292 F F F 28 560-579 166 166 166 289 289 289 F F F 29 580-599 164 164 164 287 287 287 F F F Max gen: 167, Max flush: 292 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.66496035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.711083264Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.711746454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.711788514Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.711803081Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.711821527Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856} {"msg":"Crucible a029c272-903b-4226-bd7e-be77c086e659 has session id: 0eca7967-05c3-4ea2-9fd8-27507e7e63a0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.712442906Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Upstairs opts: Upstairs UUID: a029c272-903b-4226-bd7e-be77c086e659, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.712491055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Crucible stats registered with UUID: a029c272-903b-4226-bd7e-be77c086e659","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.712516479Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:36.712537374Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713035482Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713069744Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856} {"msg":"a029c272-903b-4226-bd7e-be77c086e659 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713142628Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.71317546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"0","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713212992Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"0","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713229598Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"1","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713243914Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"1","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713257371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"2","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713271758Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"2","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713295133Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713328595Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713359328Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40249)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713409117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59281)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713450438Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:45210)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.713513943Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.714212985Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"0","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.714242079Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"2","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.714257505Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"1","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.714787025Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"0","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.714850071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"1","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.714967814Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"2","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715329432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715449745Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[0]R flush_numbers[0..12]: [281, 288, 282, 284, 290, 288, 289, 291, 288, 286, 286, 290]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715500583Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[0]R generation[0..12]: [160, 165, 161, 162, 166, 165, 166, 166, 165, 163, 163, 166]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715522578Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715537695Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[1]R flush_numbers[0..12]: [281, 288, 282, 284, 290, 288, 289, 291, 288, 286, 286, 290]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715551612Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[1]R generation[0..12]: [160, 165, 161, 162, 166, 165, 166, 166, 165, 163, 163, 166]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715565139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715578786Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[2]R flush_numbers[0..12]: [281, 288, 282, 284, 290, 288, 289, 291, 288, 286, 286, 290]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715592293Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[2]R generation[0..12]: [160, 165, 161, 162, 166, 165, 166, 166, 165, 163, 163, 166]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.71560581Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715619576Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715632273Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715645061Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715657668Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715670355Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"":"downstairs","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715683562Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Wait for a query_work_queue command to finish before sending IO Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715703508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"0","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715719344Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"1","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715732551Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"client":"2","":"downstairs","session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715745438Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715757355Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"a029c272-903b-4226-bd7e-be77c086e659 is now active with session: 0eca7967-05c3-4ea2-9fd8-27507e7e63a0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715769942Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.71578173Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856,"session_id":"0eca7967-05c3-4ea2-9fd8-27507e7e63a0"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:36.715793907Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 165 len 24576 data: 14 14 14 12 14 12 03/30 Write block 394 len 40960 data: 9 8 9 9 13 11 12 14 18 19 04/30 Write block 199 len 40960 data: 13 12 11 14 11 9 9 9 11 11 05/30 Flush 06/30 Write block 54 len 28672 data: 13 11 11 9 9 9 9 07/30 Read block 247 len 20480 08/30 Read block 15 len 28672 09/30 Write block 415 len 28672 data: 17 14 14 12 12 12 11 10/30 Write block 499 len 40960 data: 14 16 17 16 14 12 11 10 9 10 11/30 Write block 173 len 36864 data: 16 14 15 16 19 17 15 15 16 12/30 Write block 239 len 8192 data: 12 14 13/30 Read block 533 len 32768 14/30 Write block 138 len 4096 data: 14 15/30 Write block 397 len 4096 data: 10 16/30 Write block 279 len 16384 data: 13 13 17 14 17/30 Read block 115 len 16384 18/30 Read block 0 len 16384 19/30 Write block 4 len 24576 data: 10 10 10 10 13 11 20/30 Write block 548 len 4096 data: 14 21/30 Write block 459 len 20480 data: 13 10 11 11 13 22/30 Write block 536 len 28672 data: 12 9 7 7 6 7 11 23/30 Flush 24/30 Write block 233 len 28672 data: 19 19 18 16 17 13 13 25/30 Write block 274 len 12288 data: 9 7 6 26/30 Write block 28 len 36864 data: 14 12 12 12 10 11 10 9 9 27/30 Read block 138 len 32768 28/30 Write block 240 len 20480 data: 15 17 18 17 14 29/30 Read block 255 len 28672 30/30 Read block 377 len 8192 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:36.735079639Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1856} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 295 295 295 F F F 1 020-039 165 165 165 288 288 288 T T T 2 040-059 168 168 168 295 295 295 F F F 3 060-079 168 168 168 295 295 295 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 165 165 165 288 288 288 F F F 6 120-139 168 168 168 295 295 295 F F F 7 140-159 166 166 166 291 291 291 F F F 8 160-179 168 168 168 295 295 295 F F F 9 180-199 168 168 168 295 295 295 F F F 10 200-219 168 168 168 294 294 294 F F F 11 220-239 168 168 168 295 295 295 T T T 12 240-259 168 168 168 295 295 295 T T T 13 260-279 168 168 168 295 295 295 T T T 14 280-299 168 168 168 295 295 295 F F F 15 300-319 167 167 167 292 292 292 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 165 165 165 288 288 288 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 168 168 168 295 295 295 F F F 22 440-459 168 168 168 295 295 295 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 168 168 168 295 295 295 F F F 25 500-519 168 168 168 295 295 295 F F F 26 520-539 168 168 168 295 295 295 F F F 27 540-559 168 168 168 295 295 295 F F F 28 560-579 166 166 166 289 289 289 F F F 29 580-599 164 164 164 287 287 287 F F F Max gen: 168, Max flush: 295 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 295 295 295 F F F 1 020-039 169 169 169 296 296 296 F F F 2 040-059 168 168 168 295 295 295 F F F 3 060-079 168 168 168 295 295 295 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 165 165 165 288 288 288 F F F 6 120-139 168 168 168 295 295 295 F F F 7 140-159 166 166 166 291 291 291 F F F 8 160-179 168 168 168 295 295 295 F F F 9 180-199 168 168 168 295 295 295 F F F 10 200-219 168 168 168 294 294 294 F F F 11 220-239 169 169 169 296 296 296 F F F 12 240-259 169 169 169 296 296 296 F F F 13 260-279 169 169 169 296 296 296 F F F 14 280-299 168 168 168 295 295 295 F F F 15 300-319 167 167 167 292 292 292 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 165 165 165 288 288 288 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 168 168 168 295 295 295 F F F 22 440-459 168 168 168 295 295 295 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 168 168 168 295 295 295 F F F 25 500-519 168 168 168 295 295 295 F F F 26 520-539 168 168 168 295 295 295 F F F 27 540-559 168 168 168 295 295 295 F F F 28 560-579 166 166 166 289 289 289 F F F 29 580-599 164 164 164 287 287 287 F F F Max gen: 169, Max flush: 296 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:40.973163823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:41.016617362Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.017010393Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.049383181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.049989234Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.050023826Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.050037263Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.05005125Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864} {"msg":"Crucible daaead42-db61-4bb1-9de2-ad505c799131 has session id: dd184db4-6169-4dc4-97d6-48cb5f84140e","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.05066886Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Upstairs opts: Upstairs UUID: daaead42-db61-4bb1-9de2-ad505c799131, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.05071298Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Crucible stats registered with UUID: daaead42-db61-4bb1-9de2-ad505c799131","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.050733755Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:41.050747312Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051161369Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051194241Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864} {"msg":"daaead42-db61-4bb1-9de2-ad505c799131 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051248719Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051280831Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"0","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051305706Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"0","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051320062Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"1","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051334269Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"1","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051347366Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"2","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051361123Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"2","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.0513742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051393366Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.05141861Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35278)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051620434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44951)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051662735Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64433)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.051691938Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.052238494Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"1","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.052276506Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"2","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.052429481Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"0","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.052878609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"1","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.052917671Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"2","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053022807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"0","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.05345078Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053486492Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[0]R flush_numbers[0..12]: [295, 296, 295, 295, 290, 288, 295, 291, 295, 295, 294, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053520124Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[0]R generation[0..12]: [168, 169, 168, 168, 166, 165, 168, 166, 168, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053558675Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.0535844Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[1]R flush_numbers[0..12]: [295, 296, 295, 295, 290, 288, 295, 291, 295, 295, 294, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053607304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[1]R generation[0..12]: [168, 169, 168, 168, 166, 165, 168, 166, 168, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053629659Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053652764Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[2]R flush_numbers[0..12]: [295, 296, 295, 295, 290, 288, 295, 291, 295, 295, 294, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053677908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[2]R generation[0..12]: [168, 169, 168, 168, 166, 165, 168, 166, 168, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053703333Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053732486Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.0537595Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:25:41.053786454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053814728Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053840652Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"":"downstairs","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053866356Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.05389061Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"0","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053916914Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"1","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053943778Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"client":"2","":"downstairs","session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.053968623Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.05398209Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"daaead42-db61-4bb1-9de2-ad505c799131 is now active with session: dd184db4-6169-4dc4-97d6-48cb5f84140e","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-15T05:25:41.053995107Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.05402387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864,"session_id":"dd184db4-6169-4dc4-97d6-48cb5f84140e"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:41.054036377Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 12288 data: 15 13 10 02/30 Read block 58 len 8192 03/30 Write block 102 len 12288 data: 18 17 14 04/30 Write block 57 len 36864 data: 10 10 10 10 12 11 13 14 16 05/30 Write block 278 len 24576 data: 13 14 14 18 15 10 06/30 Write block 439 len 4096 data: 14 07/30 Read block 232 len 20480 08/30 Read block 158 len 8192 09/30 Read block 576 len 20480 10/30 Read block 548 len 28672 11/30 Read block 202 len 8192 12/30 Flush 13/30 Write block 510 len 40960 data: 10 10 14 14 13 14 13 12 13 16 14/30 Write block 241 len 4096 data: 18 15/30 Read block 366 len 4096 16/30 Flush 17/30 Write block 447 len 16384 data: 14 12 13 14 18/30 Write block 549 len 4096 data: 15 19/30 Read block 405 len 20480 20/30 Read block 495 len 12288 21/30 Read block 247 len 24576 22/30 Read block 339 len 4096 23/30 Write block 54 len 40960 data: 14 12 12 11 11 11 11 13 12 14 24/30 Write block 1 len 4096 data: 7 25/30 Read block 120 len 8192 26/30 Write block 526 len 36864 data: 20 15 15 14 11 12 9 10 12 27/30 Read block 377 len 36864 28/30 Read block 234 len 20480 29/30 Write block 203 len 28672 data: 12 10 10 10 12 12 12 30/30 Read block 300 len 36864 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 4 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 14 13 12 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:41.072408446Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1864} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 295 295 295 T T T 1 020-039 169 169 169 296 296 296 F F F 2 040-059 170 170 170 297 297 297 T T T 3 060-079 170 170 170 297 297 297 T T T 4 080-099 166 166 166 290 290 290 F F F 5 100-119 170 170 170 297 297 297 F F F 6 120-139 168 168 168 295 295 295 F F F 7 140-159 166 166 166 291 291 291 F F F 8 160-179 168 168 168 295 295 295 F F F 9 180-199 168 168 168 295 295 295 F F F 10 200-219 168 168 168 294 294 294 T T T 11 220-239 169 169 169 296 296 296 F F F 12 240-259 170 170 170 298 298 298 F F F 13 260-279 170 170 170 297 297 297 F F F 14 280-299 170 170 170 297 297 297 F F F 15 300-319 167 167 167 292 292 292 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 165 165 165 288 288 288 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 168 168 168 295 295 295 T T T 23 460-479 168 168 168 295 295 295 F F F 24 480-499 168 168 168 295 295 295 F F F 25 500-519 170 170 170 298 298 298 F F F 26 520-539 168 168 168 295 295 295 T T T 27 540-559 168 168 168 295 295 295 T T T 28 560-579 166 166 166 289 289 289 F F F 29 580-599 164 164 164 287 287 287 F F F Max gen: 170, Max flush: 298 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 169 169 169 296 296 296 F F F 2 040-059 171 171 171 299 299 299 F F F 3 060-079 171 171 171 299 299 299 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 170 170 170 297 297 297 F F F 6 120-139 168 168 168 295 295 295 F F F 7 140-159 166 166 166 291 291 291 F F F 8 160-179 168 168 168 295 295 295 F F F 9 180-199 168 168 168 295 295 295 F F F 10 200-219 171 171 171 299 299 299 F F F 11 220-239 169 169 169 296 296 296 F F F 12 240-259 170 170 170 298 298 298 F F F 13 260-279 170 170 170 297 297 297 F F F 14 280-299 170 170 170 297 297 297 F F F 15 300-319 167 167 167 292 292 292 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 165 165 165 288 288 288 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 171 171 171 299 299 299 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 168 168 168 295 295 295 F F F 25 500-519 170 170 170 298 298 298 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 171 171 171 299 299 299 F F F 28 560-579 166 166 166 289 289 289 F F F 29 580-599 164 164 164 287 287 287 F F F Max gen: 171, Max flush: 299 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.354674404Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:45.398529071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.398910735Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.431067982Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.431690141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.431725503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.43173926Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.431758575Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872} {"msg":"Crucible 1ac084cb-3a96-4102-b608-543a5ecfb515 has session id: 723cc272-1179-4009-b867-5510a269a6b4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.43240314Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Upstairs opts: Upstairs UUID: 1ac084cb-3a96-4102-b608-543a5ecfb515, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.432464036Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Crucible stats registered with UUID: 1ac084cb-3a96-4102-b608-543a5ecfb515","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.432492339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:45.432513395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.432912324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.432945067Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872} {"msg":"1ac084cb-3a96-4102-b608-543a5ecfb515 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.432998745Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433032847Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"0","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433059171Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"0","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433075327Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"1","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433101271Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"1","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433167076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"2","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433185242Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"2","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433204648Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433227543Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433255307Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:33132)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433279991Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43044)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433304185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:52940)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.433392236Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.434046607Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"1","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.434087688Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"2","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.434118641Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"0","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.434724864Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"2","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.434889546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"1","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.434921549Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"0","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435334696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435453349Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[0]R flush_numbers[0..12]: [299, 296, 299, 299, 290, 297, 295, 291, 295, 295, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435587388Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[0]R generation[0..12]: [171, 169, 171, 171, 166, 170, 168, 166, 168, 168, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435611363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435626339Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[1]R flush_numbers[0..12]: [299, 296, 299, 299, 290, 297, 295, 291, 295, 295, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435640536Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[1]R generation[0..12]: [171, 169, 171, 171, 166, 170, 168, 166, 168, 168, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435654153Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.43566777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[2]R flush_numbers[0..12]: [299, 296, 299, 299, 290, 297, 295, 291, 295, 295, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435681397Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[2]R generation[0..12]: [171, 169, 171, 171, 166, 170, 168, 166, 168, 168, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435704012Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435730266Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.43575562Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435780575Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435811957Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435836562Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"":"downstairs","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435858877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435880432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"0","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435908306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"1","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.43593213Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"client":"2","":"downstairs","session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435954155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.43597544Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"1ac084cb-3a96-4102-b608-543a5ecfb515 is now active with session: 723cc272-1179-4009-b867-5510a269a6b4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.435998755Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:45.4360206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872,"session_id":"723cc272-1179-4009-b867-5510a269a6b4"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-15T05:25:45.436042645Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 493 len 36864 data: 6 10 10 10 11 13 15 17 18 03/30 Write block 298 len 28672 data: 3 3 3 6 7 11 12 04/30 Read block 405 len 4096 05/30 Write block 102 len 20480 data: 19 18 15 15 14 06/30 Read block 403 len 12288 07/30 Flush 08/30 Write block 411 len 8192 data: 16 16 09/30 Read block 367 len 20480 10/30 Read block 149 len 12288 11/30 Read block 314 len 16384 12/30 Write block 562 len 32768 data: 9 10 9 7 8 10 13 14 13/30 Read block 537 len 4096 14/30 Write block 539 len 32768 data: 8 7 8 12 13 14 15 14 15/30 Write block 279 len 8192 data: 15 15 16/30 Read block 65 len 4096 17/30 Write block 286 len 4096 data: 9 18/30 Write block 126 len 4096 data: 8 19/30 Read block 82 len 16384 20/30 Write block 352 len 28672 data: 13 16 16 15 19 18 15 21/30 Read block 286 len 24576 22/30 Write block 179 len 32768 data: 16 16 17 14 14 12 9 13 23/30 Read block 158 len 36864 24/30 Read block 291 len 12288 25/30 Read block 136 len 40960 26/30 Read block 433 len 24576 27/30 Write block 578 len 16384 data: 14 18 17 19 28/30 Read block 579 len 40960 29/30 Write block 185 len 4096 data: 10 30/30 Read block 598 len 4096 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 22 23 21 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:45.455045632Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1872} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 169 169 169 296 296 296 F F F 2 040-059 171 171 171 299 299 299 F F F 3 060-079 171 171 171 299 299 299 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 172 172 172 301 301 301 F F F 6 120-139 168 168 168 295 295 295 T T T 7 140-159 166 166 166 291 291 291 F F F 8 160-179 168 168 168 295 295 295 T T T 9 180-199 168 168 168 295 295 295 T T T 10 200-219 171 171 171 299 299 299 F F F 11 220-239 169 169 169 296 296 296 F F F 12 240-259 170 170 170 298 298 298 F F F 13 260-279 170 170 170 297 297 297 T T T 14 280-299 172 172 172 301 301 301 T T T 15 300-319 172 172 172 301 301 301 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 165 165 165 288 288 288 T T T 18 360-379 164 164 164 287 287 287 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 168 168 168 295 295 295 T T T 21 420-439 170 170 170 297 297 297 F F F 22 440-459 171 171 171 299 299 299 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 172 172 172 301 301 301 F F F 25 500-519 172 172 172 301 301 301 F F F 26 520-539 171 171 171 299 299 299 T T T 27 540-559 171 171 171 299 299 299 T T T 28 560-579 166 166 166 289 289 289 T T T 29 580-599 164 164 164 287 287 287 T T T Max gen: 172, Max flush: 301 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 169 169 169 296 296 296 F F F 2 040-059 171 171 171 299 299 299 F F F 3 060-079 171 171 171 299 299 299 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 172 172 172 301 301 301 F F F 6 120-139 173 173 173 302 302 302 F F F 7 140-159 166 166 166 291 291 291 F F F 8 160-179 173 173 173 302 302 302 F F F 9 180-199 173 173 173 302 302 302 F F F 10 200-219 171 171 171 299 299 299 F F F 11 220-239 169 169 169 296 296 296 F F F 12 240-259 170 170 170 298 298 298 F F F 13 260-279 173 173 173 302 302 302 F F F 14 280-299 173 173 173 302 302 302 F F F 15 300-319 172 172 172 301 301 301 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 173 173 173 302 302 302 F F F 18 360-379 164 164 164 287 287 287 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 173 173 173 302 302 302 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 171 171 171 299 299 299 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 172 172 172 301 301 301 F F F 25 500-519 172 172 172 301 301 301 F F F 26 520-539 173 173 173 302 302 302 F F F 27 540-559 173 173 173 302 302 302 F F F 28 560-579 173 173 173 302 302 302 F F F 29 580-599 173 173 173 302 302 302 F F F Max gen: 173, Max flush: 302 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.76455426Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.809836854Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.810664746Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.810709856Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.810725312Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.810739749Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880} {"msg":"Crucible 26cb3cb7-75b6-46dd-9b8e-c709faa248d0 has session id: debee8af-81fa-48bc-8235-771832f91c1d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.8113099Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Upstairs opts: Upstairs UUID: 26cb3cb7-75b6-46dd-9b8e-c709faa248d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811366767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Crucible stats registered with UUID: 26cb3cb7-75b6-46dd-9b8e-c709faa248d0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811390882Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:49.811411157Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811819045Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811855426Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880} {"msg":"26cb3cb7-75b6-46dd-9b8e-c709faa248d0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811880751Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811902666Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"0","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811933729Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"0","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811955704Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"1","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.811978669Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"1","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.81201798Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"2","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812043924Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"2","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812065429Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812087284Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812109509Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45486)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812131524Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47215)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812154069Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43655)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812177643Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812701665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"0","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.812735107Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"1","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.813322554Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"1","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.813480928Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"0","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.813899434Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.81400198Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.814288305Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"2","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.81479823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"2","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[0]R flush_numbers[0..12]: [299, 296, 299, 299, 290, 301, 302, 291, 302, 302, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815301286Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[0]R generation[0..12]: [171, 169, 171, 171, 166, 172, 173, 166, 173, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.81532618Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815356683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[1]R flush_numbers[0..12]: [299, 296, 299, 299, 290, 301, 302, 291, 302, 302, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.81537155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[1]R generation[0..12]: [171, 169, 171, 171, 166, 172, 173, 166, 173, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815385207Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815398894Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[2]R flush_numbers[0..12]: [299, 296, 299, 299, 290, 301, 302, 291, 302, 302, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815412251Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[2]R generation[0..12]: [171, 169, 171, 171, 166, 172, 173, 166, 173, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815430117Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815444304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815456841Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815469708Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815489503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.81550264Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"":"downstairs","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815515328Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815527155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"0","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815540042Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"1","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815552709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"client":"2","":"downstairs","session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815565546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815577473Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"26cb3cb7-75b6-46dd-9b8e-c709faa248d0 is now active with session: debee8af-81fa-48bc-8235-771832f91c1d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.81559002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815601658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880,"session_id":"debee8af-81fa-48bc-8235-771832f91c1d"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:49.815613525Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 499 len 16384 data: 16 18 19 17 03/30 Read block 461 len 36864 04/30 Flush 05/30 Flush 06/30 Write block 25 len 16384 data: 10 13 13 15 07/30 Read block 454 len 28672 08/30 Flush 09/30 Write block 109 len 16384 data: 12 9 11 11 10/30 Flush 11/30 Write block 293 len 40960 data: 10 8 6 3 3 4 4 4 7 8 12/30 Read block 59 len 28672 13/30 Read block 492 len 8192 14/30 Write block 271 len 12288 data: 8 7 8 15/30 Flush 16/30 Write block 259 len 12288 data: 16 15 15 17/30 Read block 406 len 32768 18/30 Read block 120 len 20480 19/30 Read block 9 len 40960 20/30 Write block 268 len 20480 data: 17 13 10 9 8 21/30 Write block 72 len 8192 data: 11 10 22/30 Write block 293 len 8192 data: 11 9 23/30 Read block 7 len 24576 24/30 Write block 175 len 12288 data: 16 17 20 25/30 Write block 134 len 8192 data: 9 11 26/30 Read block 520 len 4096 27/30 Write block 357 len 36864 data: 19 16 10 11 13 13 13 17 17 28/30 Write block 122 len 12288 data: 9 10 9 29/30 Write block 45 len 28672 data: 16 12 10 9 7 9 10 30/30 Read block 298 len 20480 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 3 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 14 15 15 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:49.834067246Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1880} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 174 174 174 306 306 306 F F F 2 040-059 171 171 171 299 299 299 T T T 3 060-079 171 171 171 299 299 299 T T T 4 080-099 166 166 166 290 290 290 F F F 5 100-119 174 174 174 307 307 307 F F F 6 120-139 173 173 173 302 302 302 T T T 7 140-159 166 166 166 291 291 291 F F F 8 160-179 173 173 173 302 302 302 T T T 9 180-199 173 173 173 302 302 302 F F F 10 200-219 171 171 171 299 299 299 F F F 11 220-239 169 169 169 296 296 296 F F F 12 240-259 170 170 170 298 298 298 T T T 13 260-279 174 174 174 308 308 308 T T T 14 280-299 174 174 174 308 308 308 T T T 15 300-319 174 174 174 308 308 308 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 173 173 173 302 302 302 T T T 18 360-379 164 164 164 287 287 287 T T T 19 380-399 168 168 168 295 295 295 F F F 20 400-419 173 173 173 302 302 302 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 171 171 171 299 299 299 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 174 174 174 304 304 304 F F F 25 500-519 174 174 174 304 304 304 F F F 26 520-539 173 173 173 302 302 302 F F F 27 540-559 173 173 173 302 302 302 F F F 28 560-579 173 173 173 302 302 302 F F F 29 580-599 173 173 173 302 302 302 F F F Max gen: 174, Max flush: 308 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 174 174 174 306 306 306 F F F 2 040-059 175 175 175 309 309 309 F F F 3 060-079 175 175 175 309 309 309 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 174 174 174 307 307 307 F F F 6 120-139 175 175 175 309 309 309 F F F 7 140-159 166 166 166 291 291 291 F F F 8 160-179 175 175 175 309 309 309 F F F 9 180-199 173 173 173 302 302 302 F F F 10 200-219 171 171 171 299 299 299 F F F 11 220-239 169 169 169 296 296 296 F F F 12 240-259 175 175 175 309 309 309 F F F 13 260-279 175 175 175 309 309 309 F F F 14 280-299 175 175 175 309 309 309 F F F 15 300-319 174 174 174 308 308 308 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 175 175 175 309 309 309 F F F 18 360-379 175 175 175 309 309 309 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 173 173 173 302 302 302 F F F 21 420-439 170 170 170 297 297 297 F F F 22 440-459 171 171 171 299 299 299 F F F 23 460-479 168 168 168 295 295 295 F F F 24 480-499 174 174 174 304 304 304 F F F 25 500-519 174 174 174 304 304 304 F F F 26 520-539 173 173 173 302 302 302 F F F 27 540-559 173 173 173 302 302 302 F F F 28 560-579 173 173 173 302 302 302 F F F 29 580-599 173 173 173 302 302 302 F F F Max gen: 175, Max flush: 309 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.106397194Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.152448944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.153112944Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.153180308Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.153206972Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.153240635Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888} {"msg":"Crucible 94f263fb-fd38-456e-a4fc-1f8440aa40a0 has session id: 12184803-68f5-407d-9b50-4b50d4834f0a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.153785901Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Upstairs opts: Upstairs UUID: 94f263fb-fd38-456e-a4fc-1f8440aa40a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.153832101Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Crucible stats registered with UUID: 94f263fb-fd38-456e-a4fc-1f8440aa40a0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.153856035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:54.153875371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154334827Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154358102Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888} {"msg":"94f263fb-fd38-456e-a4fc-1f8440aa40a0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154418188Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154435584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"0","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154460728Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"0","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154485642Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"1","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154512627Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"1","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.15453919Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"2","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154565564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"2","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154592418Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154626861Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154681628Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64130)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.15471551Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62614)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154835513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37113)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.154862287Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.155448744Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"2","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.155479608Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"0","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.155509261Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"1","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156113664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"2","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156142637Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"0","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156195165Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"1","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.15666141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156688314Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[0]R flush_numbers[0..12]: [299, 306, 309, 309, 290, 307, 309, 291, 309, 302, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156813985Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[0]R generation[0..12]: [171, 174, 175, 175, 166, 174, 175, 166, 175, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.15683693Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156851987Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[1]R flush_numbers[0..12]: [299, 306, 309, 309, 290, 307, 309, 291, 309, 302, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156865803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[1]R generation[0..12]: [171, 174, 175, 175, 166, 174, 175, 166, 175, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.15687908Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156892787Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[2]R flush_numbers[0..12]: [299, 306, 309, 309, 290, 307, 309, 291, 309, 302, 299, 296]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156906334Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[2]R generation[0..12]: [171, 174, 175, 175, 166, 174, 175, 166, 175, 173, 171, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156919541Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156932938Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156945545Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156958253Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:25:54.15697084Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.156987086Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"":"downstairs","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.157015569Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.157028737Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"0","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.157044473Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"1","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.15705771Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"client":"2","":"downstairs","session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.157070587Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.157082554Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"94f263fb-fd38-456e-a4fc-1f8440aa40a0 is now active with session: 12184803-68f5-407d-9b50-4b50d4834f0a","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.157095051Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.157106609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888,"session_id":"12184803-68f5-407d-9b50-4b50d4834f0a"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:54.157118506Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 24576 data: 7 11 14 14 16 13 02/30 Read block 366 len 16384 03/30 Read block 406 len 20480 04/30 Read block 354 len 40960 05/30 Read block 144 len 16384 06/30 Read block 170 len 36864 07/30 Write block 198 len 12288 data: 12 14 13 08/30 Flush 09/30 Write block 367 len 24576 data: 12 13 10 11 10 8 10/30 Write block 200 len 24576 data: 14 12 15 13 11 11 11/30 Write block 166 len 36864 data: 15 15 13 15 13 12 13 17 15 12/30 Read block 246 len 32768 13/30 Write block 237 len 20480 data: 18 14 14 16 19 14/30 Read block 400 len 8192 15/30 Flush 16/30 Write block 512 len 20480 data: 15 15 14 15 14 17/30 Read block 260 len 8192 18/30 Read block 356 len 24576 19/30 Read block 370 len 32768 20/30 Write block 421 len 36864 data: 12 10 12 13 14 15 13 12 14 21/30 Write block 405 len 36864 data: 18 13 13 13 11 14 17 17 18 22/30 Read block 314 len 36864 23/30 Flush 24/30 Write block 247 len 12288 data: 12 13 12 25/30 Write block 477 len 12288 data: 10 8 13 26/30 Write block 254 len 4096 data: 10 27/30 Write block 448 len 12288 data: 13 14 15 28/30 Read block 35 len 4096 29/30 Write block 143 len 24576 data: 18 21 20 22 19 17 30/30 Write block 208 len 12288 data: 13 13 9 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:54.176028403Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1888} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 176 176 176 310 310 310 F F F 2 040-059 175 175 175 309 309 309 F F F 3 060-079 175 175 175 309 309 309 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 174 174 174 307 307 307 F F F 6 120-139 175 175 175 309 309 309 F F F 7 140-159 166 166 166 291 291 291 T T T 8 160-179 176 176 176 311 311 311 F F F 9 180-199 176 176 176 310 310 310 F F F 10 200-219 176 176 176 311 311 311 T T T 11 220-239 176 176 176 311 311 311 F F F 12 240-259 176 176 176 311 311 311 T T T 13 260-279 175 175 175 309 309 309 F F F 14 280-299 175 175 175 309 309 309 F F F 15 300-319 174 174 174 308 308 308 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 175 175 175 309 309 309 F F F 18 360-379 176 176 176 311 311 311 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 176 176 176 312 312 312 F F F 21 420-439 176 176 176 312 312 312 F F F 22 440-459 171 171 171 299 299 299 T T T 23 460-479 168 168 168 295 295 295 T T T 24 480-499 174 174 174 304 304 304 F F F 25 500-519 176 176 176 312 312 312 F F F 26 520-539 173 173 173 302 302 302 F F F 27 540-559 173 173 173 302 302 302 F F F 28 560-579 173 173 173 302 302 302 F F F 29 580-599 173 173 173 302 302 302 F F F Max gen: 176, Max flush: 312 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 176 176 176 310 310 310 F F F 2 040-059 175 175 175 309 309 309 F F F 3 060-079 175 175 175 309 309 309 F F F 4 080-099 166 166 166 290 290 290 F F F 5 100-119 174 174 174 307 307 307 F F F 6 120-139 175 175 175 309 309 309 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 176 176 176 311 311 311 F F F 9 180-199 176 176 176 310 310 310 F F F 10 200-219 177 177 177 313 313 313 F F F 11 220-239 176 176 176 311 311 311 F F F 12 240-259 177 177 177 313 313 313 F F F 13 260-279 175 175 175 309 309 309 F F F 14 280-299 175 175 175 309 309 309 F F F 15 300-319 174 174 174 308 308 308 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 175 175 175 309 309 309 F F F 18 360-379 176 176 176 311 311 311 F F F 19 380-399 168 168 168 295 295 295 F F F 20 400-419 176 176 176 312 312 312 F F F 21 420-439 176 176 176 312 312 312 F F F 22 440-459 177 177 177 313 313 313 F F F 23 460-479 177 177 177 313 313 313 F F F 24 480-499 174 174 174 304 304 304 F F F 25 500-519 176 176 176 312 312 312 F F F 26 520-539 173 173 173 302 302 302 F F F 27 540-559 173 173 173 302 302 302 F F F 28 560-579 173 173 173 302 302 302 F F F 29 580-599 173 173 173 302 302 302 F F F Max gen: 177, Max flush: 313 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.422776696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:58.467184458Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.46761931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.499925353Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.500543183Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.500586424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.500604669Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.500622236Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896} {"msg":"Crucible 32c3640a-5a29-4ec0-9065-66516688e482 has session id: 1411a103-8cb0-42ad-9b2c-124b6e86654b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501217661Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Upstairs opts: Upstairs UUID: 32c3640a-5a29-4ec0-9065-66516688e482, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501306761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Crucible stats registered with UUID: 32c3640a-5a29-4ec0-9065-66516688e482","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501335504Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:58.50135118Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501798809Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501832521Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896} {"msg":"32c3640a-5a29-4ec0-9065-66516688e482 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501885329Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501915803Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"0","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501940177Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"0","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501955363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"1","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.501981338Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"1","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502007242Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"2","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502033306Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"2","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.50205983Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502085144Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502109679Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46079)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502139102Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"ds_connection connected from Ok(127.0.0.1:57245)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502167575Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39552)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.50236664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502825946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"1","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502865557Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"2","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.502895301Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"0","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.503483757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"0","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.503511891Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"1","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.503646851Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"2","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504083861Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504188608Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[0]R flush_numbers[0..12]: [299, 310, 309, 309, 290, 307, 309, 313, 311, 310, 313, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504250384Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[0]R generation[0..12]: [171, 176, 175, 175, 166, 174, 175, 177, 176, 176, 177, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504269709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504283916Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[1]R flush_numbers[0..12]: [299, 310, 309, 309, 290, 307, 309, 313, 311, 310, 313, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504297703Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[1]R generation[0..12]: [171, 176, 175, 175, 166, 174, 175, 177, 176, 176, 177, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.50431158Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504325607Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[2]R flush_numbers[0..12]: [299, 310, 309, 309, 290, 307, 309, 313, 311, 310, 313, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504339164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[2]R generation[0..12]: [171, 176, 175, 175, 166, 174, 175, 177, 176, 176, 177, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504352441Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504369587Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504382504Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504395361Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504407968Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504420435Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"":"downstairs","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504433092Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504451408Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"0","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504464535Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"1","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504477112Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"client":"2","":"downstairs","session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504489929Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504501877Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"32c3640a-5a29-4ec0-9065-66516688e482 is now active with session: 1411a103-8cb0-42ad-9b2c-124b6e86654b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504514424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504525981Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896,"session_id":"1411a103-8cb0-42ad-9b2c-124b6e86654b"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:25:58.504537739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 128 len 24576 data: 8 8 9 10 11 11 02/30 Read block 33 len 16384 03/30 Write block 438 len 16384 data: 15 15 15 14 04/30 Write block 56 len 16384 data: 13 12 12 12 05/30 Flush 06/30 Write block 502 len 24576 data: 18 15 13 12 11 10 07/30 Read block 148 len 32768 08/30 Write block 585 len 4096 data: 16 09/30 Read block 191 len 12288 10/30 Write block 183 len 32768 data: 15 13 11 14 15 15 16 17 11/30 Write block 25 len 28672 data: 12 15 15 17 14 13 13 12/30 Write block 78 len 36864 data: 10 11 12 17 19 19 18 13 14 13/30 Read block 506 len 24576 14/30 Flush 15/30 Flush 16/30 Write block 301 len 12288 data: 8 9 12 17/30 Write block 460 len 20480 data: 11 12 12 14 13 18/30 Read block 396 len 4096 19/30 Read block 177 len 4096 20/30 Read block 530 len 36864 21/30 Flush 22/30 Write block 379 len 24576 data: 16 16 13 13 12 13 23/30 Read block 525 len 28672 24/30 Flush 25/30 Write block 381 len 32768 data: 14 14 13 14 12 13 12 15 26/30 Read block 363 len 20480 27/30 Write block 207 len 16384 data: 13 14 14 10 28/30 Read block 523 len 20480 29/30 Read block 124 len 36864 30/30 Write block 544 len 40960 data: 15 16 15 14 15 16 13 15 11 8 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:25:58.524795631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1896} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 178 178 178 315 315 315 F F F 2 040-059 178 178 178 314 314 314 F F F 3 060-079 178 178 178 315 315 315 F F F 4 080-099 178 178 178 315 315 315 F F F 5 100-119 174 174 174 307 307 307 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 176 176 176 311 311 311 F F F 9 180-199 178 178 178 315 315 315 F F F 10 200-219 177 177 177 313 313 313 T T T 11 220-239 176 176 176 311 311 311 F F F 12 240-259 177 177 177 313 313 313 F F F 13 260-279 175 175 175 309 309 309 F F F 14 280-299 175 175 175 309 309 309 F F F 15 300-319 178 178 178 317 317 317 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 175 175 175 309 309 309 F F F 18 360-379 178 178 178 318 318 318 F F F 19 380-399 178 178 178 318 318 318 T T T 20 400-419 176 176 176 312 312 312 F F F 21 420-439 178 178 178 314 314 314 F F F 22 440-459 178 178 178 314 314 314 F F F 23 460-479 178 178 178 317 317 317 F F F 24 480-499 174 174 174 304 304 304 F F F 25 500-519 178 178 178 315 315 315 F F F 26 520-539 173 173 173 302 302 302 F F F 27 540-559 173 173 173 302 302 302 T T T 28 560-579 173 173 173 302 302 302 F F F 29 580-599 178 178 178 315 315 315 F F F Max gen: 178, Max flush: 318 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 178 178 178 315 315 315 F F F 2 040-059 178 178 178 314 314 314 F F F 3 060-079 178 178 178 315 315 315 F F F 4 080-099 178 178 178 315 315 315 F F F 5 100-119 174 174 174 307 307 307 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 176 176 176 311 311 311 F F F 9 180-199 178 178 178 315 315 315 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 176 176 176 311 311 311 F F F 12 240-259 177 177 177 313 313 313 F F F 13 260-279 175 175 175 309 309 309 F F F 14 280-299 175 175 175 309 309 309 F F F 15 300-319 178 178 178 317 317 317 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 175 175 175 309 309 309 F F F 18 360-379 178 178 178 318 318 318 F F F 19 380-399 179 179 179 319 319 319 F F F 20 400-419 176 176 176 312 312 312 F F F 21 420-439 178 178 178 314 314 314 F F F 22 440-459 178 178 178 314 314 314 F F F 23 460-479 178 178 178 317 317 317 F F F 24 480-499 174 174 174 304 304 304 F F F 25 500-519 178 178 178 315 315 315 F F F 26 520-539 173 173 173 302 302 302 F F F 27 540-559 179 179 179 319 319 319 F F F 28 560-579 173 173 173 302 302 302 F F F 29 580-599 178 178 178 315 315 315 F F F Max gen: 179, Max flush: 319 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.785481815Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.831144543Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.831935504Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.831976305Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.831991611Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.832006178Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904} {"msg":"Crucible 1f450220-9035-474c-8f35-303b9fc34fa0 has session id: 853d42f9-76c3-45f6-bef8-8c396050ad0d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.832579028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Upstairs opts: Upstairs UUID: 1f450220-9035-474c-8f35-303b9fc34fa0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.832625028Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Crucible stats registered with UUID: 1f450220-9035-474c-8f35-303b9fc34fa0","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.832652802Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:02.832673457Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833085124Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833118296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904} {"msg":"1f450220-9035-474c-8f35-303b9fc34fa0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833171924Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833195439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"0","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833225222Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"0","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833251756Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"1","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.83327742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"1","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833301835Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"2","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833327119Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"2","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833350683Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833374898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833390155Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44866)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833408141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51852)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833439003Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56381)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.833464878Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.834245931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"2","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.834274934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"0","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.834290751Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"1","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.834878328Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"1","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.83495638Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"0","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835074253Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"2","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835362728Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835472863Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[0]R flush_numbers[0..12]: [299, 315, 314, 315, 315, 307, 314, 313, 311, 315, 319, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.83557631Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[0]R generation[0..12]: [171, 178, 178, 178, 178, 174, 178, 177, 176, 178, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835597835Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835612851Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[1]R flush_numbers[0..12]: [299, 315, 314, 315, 315, 307, 314, 313, 311, 315, 319, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835627098Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[1]R generation[0..12]: [171, 178, 178, 178, 178, 174, 178, 177, 176, 178, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835646044Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.83566066Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[2]R flush_numbers[0..12]: [299, 315, 314, 315, 315, 307, 314, 313, 311, 315, 319, 311]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835674537Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[2]R generation[0..12]: [171, 178, 178, 178, 178, 174, 178, 177, 176, 178, 179, 176]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835688044Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835701761Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835714448Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835727305Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835739933Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835758538Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"":"downstairs","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835772185Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835783912Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"0","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835796989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"1","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835809837Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"client":"2","":"downstairs","session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835822724Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835834741Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"1f450220-9035-474c-8f35-303b9fc34fa0 is now active with session: 853d42f9-76c3-45f6-bef8-8c396050ad0d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835849438Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pidDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835865124Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904,"session_id":"853d42f9-76c3-45f6-bef8-8c396050ad0d"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:02.835877001Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 571 len 32768 data: 16 18 15 13 14 15 15 15 02/30 Write block 473 len 32768 data: 12 12 12 13 11 9 14 13 03/30 Read block 191 len 28672 04/30 Read block 489 len 12288 05/30 Write block 25 len 12288 data: 13 16 16 06/30 Write block 478 len 4096 data: 10 07/30 Read block 262 len 36864 08/30 Write block 396 len 8192 data: 10 11 09/30 Read block 171 len 28672 10/30 Write block 395 len 20480 data: 9 11 12 14 12 11/30 Write block 264 len 12288 data: 15 15 16 12/30 Write block 405 len 40960 data: 19 14 14 14 12 15 18 18 19 16 13/30 Read block 48 len 40960 14/30 Write block 282 len 32768 data: 16 11 9 8 10 10 9 12 15/30 Write block 94 len 36864 data: 13 15 16 18 15 14 14 16 20 16/30 Read block 195 len 20480 17/30 Read block 437 len 8192 18/30 Write block 117 len 8192 data: 16 15 19/30 Read block 82 len 24576 20/30 Write block 174 len 4096 data: 16 21/30 Read block 178 len 12288 22/30 Write block 301 len 8192 data: 9 10 23/30 Read block 128 len 8192 24/30 Write block 220 len 16384 data: 13 16 20 26 25/30 Read block 551 len 16384 26/30 Write block 516 len 12288 data: 15 13 14 27/30 Read block 489 len 32768 28/30 Read block 325 len 12288 29/30 Read block 322 len 40960 30/30 Read block 355 len 12288 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:02.85411726Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1904} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 178 178 178 315 315 315 T T T 2 040-059 178 178 178 314 314 314 F F F 3 060-079 178 178 178 315 315 315 F F F 4 080-099 178 178 178 315 315 315 T T T 5 100-119 174 174 174 307 307 307 T T T 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 176 176 176 311 311 311 T T T 9 180-199 178 178 178 315 315 315 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 176 176 176 311 311 311 T T T 12 240-259 177 177 177 313 313 313 F F F 13 260-279 175 175 175 309 309 309 T T T 14 280-299 175 175 175 309 309 309 T T T 15 300-319 178 178 178 317 317 317 T T T 16 320-339 167 167 167 292 292 292 F F F 17 340-359 175 175 175 309 309 309 F F F 18 360-379 178 178 178 318 318 318 F F F 19 380-399 179 179 179 319 319 319 T T T 20 400-419 176 176 176 312 312 312 T T T 21 420-439 178 178 178 314 314 314 F F F 22 440-459 178 178 178 314 314 314 F F F 23 460-479 178 178 178 317 317 317 T T T 24 480-499 174 174 174 304 304 304 T T T 25 500-519 178 178 178 315 315 315 T T T 26 520-539 173 173 173 302 302 302 F F F 27 540-559 179 179 179 319 319 319 F F F 28 560-579 173 173 173 302 302 302 T T T 29 580-599 178 178 178 315 315 315 F F F Max gen: 179, Max flush: 319 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 180 180 180 320 320 320 F F F 2 040-059 178 178 178 314 314 314 F F F 3 060-079 178 178 178 315 315 315 F F F 4 080-099 180 180 180 320 320 320 F F F 5 100-119 180 180 180 320 320 320 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 180 180 180 320 320 320 F F F 9 180-199 178 178 178 315 315 315 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 180 180 180 320 320 320 F F F 12 240-259 177 177 177 313 313 313 F F F 13 260-279 180 180 180 320 320 320 F F F 14 280-299 180 180 180 320 320 320 F F F 15 300-319 180 180 180 320 320 320 F F F 16 320-339 167 167 167 292 292 292 F F F 17 340-359 175 175 175 309 309 309 F F F 18 360-379 178 178 178 318 318 318 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 180 180 180 320 320 320 F F F 21 420-439 178 178 178 314 314 314 F F F 22 440-459 178 178 178 314 314 314 F F F 23 460-479 180 180 180 320 320 320 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 180 180 180 320 320 320 F F F 26 520-539 173 173 173 302 302 302 F F F 27 540-559 179 179 179 319 319 319 F F F 28 560-579 180 180 180 320 320 320 F F F 29 580-599 178 178 178 315 315 315 F F F Max gen: 180, Max flush: 320 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.14700358Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:17.190730757Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.191097904Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.223619008Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.22436135Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.22440476Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.224421776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.224454129Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912} {"msg":"Crucible d4adbb9a-1ff3-4d77-84c7-e581e9b7646c has session id: 73914e5c-6ec7-4085-8ecc-7e0c8c982fdd","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225080747Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Upstairs opts: Upstairs UUID: d4adbb9a-1ff3-4d77-84c7-e581e9b7646c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225139094Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Crucible stats registered with UUID: d4adbb9a-1ff3-4d77-84c7-e581e9b7646c","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225169067Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:17.225193752Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225558289Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.22559739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912} {"msg":"d4adbb9a-1ff3-4d77-84c7-e581e9b7646c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225657387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.22568459Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"0","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225723472Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"0","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225753435Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"1","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225782588Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"1","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225869708Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"2","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225919977Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"2","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225949141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225974105Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.225997619Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36821)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.226148145Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:59131)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.226187156Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41279)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.226213071Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.226621018Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"1","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.226664699Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"2","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.226794769Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"0","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.227307313Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"1","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.227334747Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"2","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.227356042Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"0","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.227887762Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.227912206Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[0]R flush_numbers[0..12]: [299, 320, 314, 315, 320, 320, 314, 313, 320, 315, 319, 320]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.227969613Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[0]R generation[0..12]: [171, 180, 178, 178, 180, 180, 178, 177, 180, 178, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.227988879Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.22802694Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[1]R flush_numbers[0..12]: [299, 320, 314, 315, 320, 320, 314, 313, 320, 315, 319, 320]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228042176Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[1]R generation[0..12]: [171, 180, 178, 178, 180, 180, 178, 177, 180, 178, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228055713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.22806938Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[2]R flush_numbers[0..12]: [299, 320, 314, 315, 320, 320, 314, 313, 320, 315, 319, 320]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228082807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[2]R generation[0..12]: [171, 180, 178, 178, 180, 180, 178, 177, 180, 178, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228100223Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.22811443Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Max found gen is 181","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228133756Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Generation requested: 182 >= found:181","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228147013Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.22815956Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228172237Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"":"downstairs","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228184884Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228196321Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"0","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228209329Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"1","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228222076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"client":"2","":"downstairs","session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228234893Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.2282467Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"d4adbb9a-1ff3-4d77-84c7-e581e9b7646c is now active with session: 73914e5c-6ec7-4085-8ecc-7e0c8c982fdd","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228259257Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228271035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912,"session_id":"73914e5c-6ec7-4085-8ecc-7e0c8c982fdd"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:17.228283172Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 29 len 28672 data: 15 14 14 11 12 11 10 02/30 Write block 330 len 20480 data: 10 9 9 9 13 03/30 Write block 191 len 12288 data: 17 17 14 04/30 Write block 581 len 28672 data: 20 19 19 16 17 14 13 05/30 Write block 553 len 24576 data: 9 8 10 9 10 8 06/30 Flush 07/30 Write block 522 len 28672 data: 19 19 20 19 21 16 16 08/30 Read block 17 len 16384 09/30 Write block 357 len 24576 data: 20 17 11 12 14 14 10/30 Read block 547 len 40960 11/30 Write block 53 len 20480 data: 12 15 13 14 13 12/30 Read block 394 len 40960 13/30 Read block 426 len 8192 14/30 Write block 522 len 24576 data: 20 20 21 20 22 17 15/30 Read block 563 len 20480 16/30 Read block 76 len 36864 17/30 Write block 372 len 28672 data: 9 12 11 13 13 16 15 18/30 Flush 19/30 Write block 21 len 20480 data: 12 9 8 8 14 20/30 Write block 52 len 32768 data: 11 13 16 14 15 14 13 13 21/30 Flush 22/30 Write block 241 len 28672 data: 20 19 18 15 14 14 13 23/30 Read block 108 len 12288 24/30 Read block 483 len 36864 25/30 Read block 392 len 16384 26/30 Flush 27/30 Write block 421 len 28672 data: 13 11 13 14 15 16 14 28/30 Read block 282 len 28672 29/30 Read block 71 len 36864 30/30 Read block 597 len 8192 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:17.248885026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1912} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 182 182 182 323 323 323 F F F 2 040-059 182 182 182 323 323 323 F F F 3 060-079 178 178 178 315 315 315 F F F 4 080-099 180 180 180 320 320 320 F F F 5 100-119 180 180 180 320 320 320 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 180 180 180 320 320 320 F F F 9 180-199 182 182 182 321 321 321 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 180 180 180 320 320 320 F F F 12 240-259 182 182 182 324 324 324 F F F 13 260-279 180 180 180 320 320 320 F F F 14 280-299 180 180 180 320 320 320 F F F 15 300-319 180 180 180 320 320 320 F F F 16 320-339 182 182 182 321 321 321 F F F 17 340-359 182 182 182 322 322 322 F F F 18 360-379 182 182 182 322 322 322 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 180 180 180 320 320 320 F F F 21 420-439 178 178 178 314 314 314 T T T 22 440-459 178 178 178 314 314 314 F F F 23 460-479 180 180 180 320 320 320 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 180 180 180 320 320 320 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 182 182 182 321 321 321 F F F 28 560-579 180 180 180 320 320 320 F F F 29 580-599 182 182 182 321 321 321 F F F Max gen: 182, Max flush: 324 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 182 182 182 323 323 323 F F F 2 040-059 182 182 182 323 323 323 F F F 3 060-079 178 178 178 315 315 315 F F F 4 080-099 180 180 180 320 320 320 F F F 5 100-119 180 180 180 320 320 320 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 180 180 180 320 320 320 F F F 9 180-199 182 182 182 321 321 321 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 180 180 180 320 320 320 F F F 12 240-259 182 182 182 324 324 324 F F F 13 260-279 180 180 180 320 320 320 F F F 14 280-299 180 180 180 320 320 320 F F F 15 300-319 180 180 180 320 320 320 F F F 16 320-339 182 182 182 321 321 321 F F F 17 340-359 182 182 182 322 322 322 F F F 18 360-379 182 182 182 322 322 322 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 180 180 180 320 320 320 F F F 21 420-439 183 183 183 325 325 325 F F F 22 440-459 178 178 178 314 314 314 F F F 23 460-479 180 180 180 320 320 320 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 180 180 180 320 320 320 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 182 182 182 321 321 321 F F F 28 560-579 180 180 180 320 320 320 F F F 29 580-599 182 182 182 321 321 321 F F F Max gen: 183, Max flush: 325 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.495826304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.54207232Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.542885006Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.542933035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.542948411Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.542962808Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920} {"msg":"Crucible a6bbb25a-e066-4ba6-8afd-bea0f3aacc58 has session id: aa2e0ace-e08c-45d0-bf14-cb85df4e5e92","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.543670888Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Upstairs opts: Upstairs UUID: a6bbb25a-e066-4ba6-8afd-bea0f3aacc58, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.543711779Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Crucible stats registered with UUID: a6bbb25a-e066-4ba6-8afd-bea0f3aacc58","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.543728395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:21.543741232Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544239439Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544264244Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920} {"msg":"a6bbb25a-e066-4ba6-8afd-bea0f3aacc58 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.54432382Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544341946Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"0","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544364741Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"0","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544378987Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"1","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544392954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"1","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544406101Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"2","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544420038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"2","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544439794Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544462648Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544486123Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"ds_connection connected from Ok(127.0.0.1:36078)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544517146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"ds_connection connected from Ok(127.0.0.1:50329)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.54454337Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"ds_connection connected from Ok(127.0.0.1:64853)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.544581841Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.545374592Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"2","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.545403655Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"0","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.545935235Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"1","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.546053508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"0","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.546082692Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"2","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.546555934Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"1","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.546763947Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.54679895Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[0]R flush_numbers[0..12]: [299, 323, 323, 315, 320, 320, 314, 313, 320, 321, 319, 320]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547098591Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[0]R generation[0..12]: [171, 182, 182, 178, 180, 180, 178, 177, 180, 182, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547119137Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547134134Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[1]R flush_numbers[0..12]: [299, 323, 323, 315, 320, 320, 314, 313, 320, 321, 319, 320]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547153619Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[1]R generation[0..12]: [171, 182, 182, 178, 180, 180, 178, 177, 180, 182, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547167726Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547181863Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[2]R flush_numbers[0..12]: [299, 323, 323, 315, 320, 320, 314, 313, 320, 321, 319, 320]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.54719543Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[2]R generation[0..12]: [171, 182, 182, 178, 180, 180, 178, 177, 180, 182, 179, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547208897Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547222503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547235131Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547248148Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547260695Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547273252Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"":"downstairs","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-15T05:26:21.547286169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547304525Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"0","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547317792Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"1","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547330849Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"client":"2","":"downstairs","session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547355173Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.54737055Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"a6bbb25a-e066-4ba6-8afd-bea0f3aacc58 is now active with session: aa2e0ace-e08c-45d0-bf14-cb85df4e5e92","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547383227Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547394995Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920,"session_id":"aa2e0ace-e08c-45d0-bf14-cb85df4e5e92"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:21.547406882Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 95 len 16384 data: 16 17 19 16 02/30 Read block 24 len 28672 03/30 Read block 211 len 32768 04/30 Read block 156 len 16384 05/30 Read block 406 len 36864 06/30 Flush 07/30 Write block 77 len 8192 data: 10 11 08/30 Write block 512 len 24576 data: 16 16 15 16 16 14 09/30 Write block 367 len 28672 data: 13 14 11 12 11 10 13 10/30 Flush 11/30 Write block 246 len 28672 data: 15 14 14 13 10 12 9 12/30 Read block 225 len 28672 13/30 Read block 48 len 28672 14/30 Read block 284 len 8192 15/30 Write block 318 len 40960 data: 12 14 16 20 18 15 13 9 9 9 16/30 Write block 587 len 36864 data: 14 11 12 12 12 10 9 9 5 17/30 Read block 550 len 12288 18/30 Write block 163 len 24576 data: 19 14 15 16 16 14 19/30 Read block 455 len 32768 20/30 Write block 44 len 28672 data: 16 17 13 11 10 8 10 21/30 Read block 349 len 8192 22/30 Read block 450 len 36864 23/30 Read block 374 len 36864 24/30 Write block 266 len 32768 data: 17 16 18 14 11 10 9 9 25/30 Write block 227 len 12288 data: 21 20 20 26/30 Flush 27/30 Write block 230 len 4096 data: 19 28/30 Write block 445 len 40960 data: 15 13 15 14 15 16 12 12 10 12 29/30 Write block 416 len 32768 data: 15 15 13 13 13 14 12 14 30/30 Write block 404 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:21.567095093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1920} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 182 182 182 323 323 323 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 180 180 180 320 320 320 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 184 184 184 328 328 328 F F F 9 180-199 182 182 182 321 321 321 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 184 184 184 328 328 328 T T T 12 240-259 184 184 184 328 328 328 F F F 13 260-279 184 184 184 328 328 328 F F F 14 280-299 180 180 180 320 320 320 F F F 15 300-319 184 184 184 328 328 328 F F F 16 320-339 184 184 184 328 328 328 F F F 17 340-359 182 182 182 322 322 322 F F F 18 360-379 184 184 184 327 327 327 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 180 180 180 320 320 320 T T T 21 420-439 183 183 183 325 325 325 T T T 22 440-459 178 178 178 314 314 314 T T T 23 460-479 180 180 180 320 320 320 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 327 327 327 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 182 182 182 321 321 321 F F F 28 560-579 180 180 180 320 320 320 F F F 29 580-599 184 184 184 328 328 328 F F F Max gen: 184, Max flush: 328 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 182 182 182 323 323 323 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 180 180 180 320 320 320 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 184 184 184 328 328 328 F F F 9 180-199 182 182 182 321 321 321 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 184 184 184 328 328 328 F F F 13 260-279 184 184 184 328 328 328 F F F 14 280-299 180 180 180 320 320 320 F F F 15 300-319 184 184 184 328 328 328 F F F 16 320-339 184 184 184 328 328 328 F F F 17 340-359 182 182 182 322 322 322 F F F 18 360-379 184 184 184 327 327 327 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 185 185 185 329 329 329 F F F 21 420-439 185 185 185 329 329 329 F F F 22 440-459 185 185 185 329 329 329 F F F 23 460-479 180 180 180 320 320 320 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 327 327 327 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 182 182 182 321 321 321 F F F 28 560-579 180 180 180 320 320 320 F F F 29 580-599 184 184 184 328 328 328 F F F Max gen: 185, Max flush: 329 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.809502807Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.857030523Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.857745241Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.857810736Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.857840659Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.857864574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928} {"msg":"Crucible acbf2e4d-0d15-45b0-aed6-6b7f35204f2f has session id: 840102bd-b36f-481a-a3ae-be1b505a267b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.858535632Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Upstairs opts: Upstairs UUID: acbf2e4d-0d15-45b0-aed6-6b7f35204f2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.85858602Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Crucible stats registered with UUID: acbf2e4d-0d15-45b0-aed6-6b7f35204f2f","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.858613164Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:25.858634169Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859080638Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859104993Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928} {"msg":"acbf2e4d-0d15-45b0-aed6-6b7f35204f2f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859191823Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859209279Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"0","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859231604Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"0","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859245191Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"1","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859259148Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"1","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859275634Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"2","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859289981Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"2","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859303008Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859316775Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.85933676Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64755)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859520049Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33252)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859560819Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49032)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.859588863Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.860192296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"0","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.860229048Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"2","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.860655072Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"1","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.860930639Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"0","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861046093Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"2","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861335848Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"1","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861492432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861604397Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[0]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 320, 314, 313, 328, 321, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861847552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[0]R generation[0..12]: [171, 182, 184, 184, 184, 180, 178, 177, 184, 182, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861868767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861885243Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[1]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 320, 314, 313, 328, 321, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.86189928Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[1]R generation[0..12]: [171, 182, 184, 184, 184, 180, 178, 177, 184, 182, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861913057Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861927454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[2]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 320, 314, 313, 328, 321, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.86194114Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[2]R generation[0..12]: [171, 182, 184, 184, 184, 180, 178, 177, 184, 182, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861954668Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861968454Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861981002Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.861993999Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:26:25.862006836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862024592Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"":"downstairs","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862041838Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862053945Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"0","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862067012Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"1","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862079949Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"client":"2","":"downstairs","session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862093066Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862105263Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"acbf2e4d-0d15-45b0-aed6-6b7f35204f2f is now active with session: 840102bd-b36f-481a-a3ae-be1b505a267b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.86211782Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862129508Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928,"session_id":"840102bd-b36f-481a-a3ae-be1b505a267b"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:25.862141405Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 8192 data: 14 10 02/30 Read block 509 len 8192 03/30 Read block 451 len 28672 04/30 Write block 331 len 32768 data: 10 10 10 14 17 17 15 15 05/30 Read block 473 len 4096 06/30 Read block 499 len 24576 07/30 Read block 98 len 16384 08/30 Write block 277 len 40960 data: 7 14 16 16 19 17 12 10 9 11 09/30 Read block 60 len 4096 10/30 Read block 47 len 16384 11/30 Read block 180 len 4096 12/30 Read block 84 len 4096 13/30 Write block 582 len 20480 data: 20 20 17 18 15 14/30 Read block 91 len 8192 15/30 Write block 400 len 8192 data: 13 15 16/30 Read block 471 len 20480 17/30 Read block 133 len 8192 18/30 Write block 332 len 36864 data: 11 11 15 18 18 16 16 15 14 19/30 Write block 417 len 32768 data: 16 14 14 14 15 13 15 15 20/30 Read block 534 len 8192 21/30 Read block 441 len 12288 22/30 Read block 277 len 24576 23/30 Write block 103 len 4096 data: 19 24/30 Write block 191 len 28672 data: 18 18 15 18 16 16 12 25/30 Read block 519 len 4096 26/30 Read block 249 len 36864 27/30 Write block 252 len 28672 data: 10 10 11 9 15 15 15 28/30 Write block 475 len 20480 data: 13 14 12 11 15 29/30 Read block 247 len 40960 30/30 Write block 186 len 16384 data: 15 16 16 17 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:25.881152629Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1928} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 182 182 182 323 323 323 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 180 180 180 320 320 320 T T T 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 184 184 184 328 328 328 F F F 9 180-199 182 182 182 321 321 321 T T T 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 184 184 184 328 328 328 T T T 13 260-279 184 184 184 328 328 328 T T T 14 280-299 180 180 180 320 320 320 T T T 15 300-319 184 184 184 328 328 328 F F F 16 320-339 184 184 184 328 328 328 T T T 17 340-359 182 182 182 322 322 322 T T T 18 360-379 184 184 184 327 327 327 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 185 185 185 329 329 329 T T T 21 420-439 185 185 185 329 329 329 T T T 22 440-459 185 185 185 329 329 329 F F F 23 460-479 180 180 180 320 320 320 T T T 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 327 327 327 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 182 182 182 321 321 321 F F F 28 560-579 180 180 180 320 320 320 F F F 29 580-599 184 184 184 328 328 328 T T T Max gen: 185, Max flush: 329 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 182 182 182 323 323 323 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 186 186 186 330 330 330 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 F F F 8 160-179 184 184 184 328 328 328 F F F 9 180-199 186 186 186 330 330 330 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 186 186 186 330 330 330 F F F 13 260-279 186 186 186 330 330 330 F F F 14 280-299 186 186 186 330 330 330 F F F 15 300-319 184 184 184 328 328 328 F F F 16 320-339 186 186 186 330 330 330 F F F 17 340-359 186 186 186 330 330 330 F F F 18 360-379 184 184 184 327 327 327 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 186 186 186 330 330 330 F F F 21 420-439 186 186 186 330 330 330 F F F 22 440-459 185 185 185 329 329 329 F F F 23 460-479 186 186 186 330 330 330 F F F 24 480-499 180 180 180 320 320 320 F F F 25 500-519 184 184 184 327 327 327 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 182 182 182 321 321 321 F F F 28 560-579 180 180 180 320 320 320 F F F 29 580-599 186 186 186 330 330 330 F F F Max gen: 186, Max flush: 330 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.163180611Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:30.207625435Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.208021855Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.240908577Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.241525777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.241562089Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.241576566Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.241590712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936} {"msg":"Crucible 5d7c4706-8be5-4daf-82d2-8a97913a67e1 has session id: 7e8fff69-3824-48dd-bd52-18eb56e7c4cb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242135379Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Upstairs opts: Upstairs UUID: 5d7c4706-8be5-4daf-82d2-8a97913a67e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242222989Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Crucible stats registered with UUID: 5d7c4706-8be5-4daf-82d2-8a97913a67e1","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242254112Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:30.242269279Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242688534Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242708089Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936} {"msg":"5d7c4706-8be5-4daf-82d2-8a97913a67e1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.24279533Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242812366Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"0","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.24283584Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"0","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242849727Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"1","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242864014Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"1","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242877111Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"2","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242890828Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"2","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242903665Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242916932Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.242935008Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51199)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.243081455Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43803)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.243117466Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:57862)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.243142921Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.243715011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"1","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.243742765Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"2","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.243878764Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"0","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.244398296Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"1","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.244438517Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"0","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.244466011Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"2","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245028804Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245064236Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[0]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 330, 314, 313, 328, 330, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245163973Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[0]R generation[0..12]: [171, 182, 184, 184, 184, 186, 178, 177, 184, 186, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245185048Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245204954Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[1]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 330, 314, 313, 328, 330, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.24521978Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[1]R generation[0..12]: [171, 182, 184, 184, 184, 186, 178, 177, 184, 186, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245233767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245247664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[2]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 330, 314, 313, 328, 330, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245261181Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[2]R generation[0..12]: [171, 182, 184, 184, 184, 186, 178, 177, 184, 186, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245274528Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245288395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Max found gen is 187","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245301482Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Generation requested: 188 >= found:187","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245315219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245328156Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245340863Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"":"downstairs","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245360409Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245372696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"0","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245385733Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"1","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.24539851Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"client":"2","":"downstairs","session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245413806Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245426204Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"5d7c4706-8be5-4daf-82d2-8a97913a67e1 is now active with session: 7e8fff69-3824-48dd-bd52-18eb56e7c4cb","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245438921Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245450708Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936,"session_id":"7e8fff69-3824-48dd-bd52-18eb56e7c4cb"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:30.245462635Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 475 len 32768 data: 14 15 13 12 16 14 14 12 02/30 Read block 354 len 24576 03/30 Read block 233 len 16384 04/30 Write block 541 len 28672 data: 9 13 14 16 17 16 15 05/30 Read block 495 len 24576 06/30 Write block 271 len 40960 data: 11 10 10 10 8 7 8 15 17 17 07/30 Write block 102 len 32768 data: 21 20 16 16 15 15 16 13 08/30 Flush 09/30 Write block 445 len 8192 data: 16 14 10/30 Write block 247 len 36864 data: 15 15 14 11 13 11 11 12 10 11/30 Read block 52 len 24576 12/30 Read block 369 len 24576 13/30 Flush 14/30 Write block 422 len 16384 data: 14 16 16 16 15/30 Read block 566 len 20480 16/30 Read block 343 len 24576 17/30 Write block 174 len 16384 data: 17 17 18 21 18/30 Write block 337 len 8192 data: 17 17 19/30 Write block 151 len 8192 data: 10 6 20/30 Read block 289 len 40960 21/30 Write block 367 len 28672 data: 14 15 12 13 12 11 14 22/30 Read block 98 len 40960 23/30 Write block 302 len 20480 data: 11 13 13 7 10 24/30 Read block 399 len 4096 25/30 Write block 298 len 16384 data: 5 5 5 10 26/30 Read block 163 len 8192 27/30 Write block 586 len 32768 data: 16 15 12 13 13 13 11 10 28/30 Read block 82 len 16384 29/30 Read block 161 len 40960 30/30 Write block 572 len 28672 data: 19 16 14 15 16 16 16 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:30.266605627Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1936} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 182 182 182 323 323 323 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 177 177 177 313 313 313 T T T 8 160-179 184 184 184 328 328 328 T T T 9 180-199 186 186 186 330 330 330 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 188 188 188 331 331 331 T T T 15 300-319 184 184 184 328 328 328 T T T 16 320-339 186 186 186 330 330 330 T T T 17 340-359 186 186 186 330 330 330 F F F 18 360-379 184 184 184 327 327 327 T T T 19 380-399 180 180 180 320 320 320 F F F 20 400-419 186 186 186 330 330 330 F F F 21 420-439 186 186 186 330 330 330 T T T 22 440-459 188 188 188 332 332 332 F F F 23 460-479 188 188 188 331 331 331 F F F 24 480-499 188 188 188 331 331 331 F F F 25 500-519 184 184 184 327 327 327 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 180 180 180 320 320 320 T T T 29 580-599 186 186 186 330 330 330 T T T Max gen: 188, Max flush: 332 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 299 299 299 F F F 1 020-039 182 182 182 323 323 323 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 178 178 178 314 314 314 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 189 189 189 333 333 333 F F F 9 180-199 186 186 186 330 330 330 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 188 188 188 331 331 331 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 189 189 189 333 333 333 F F F 16 320-339 189 189 189 333 333 333 F F F 17 340-359 186 186 186 330 330 330 F F F 18 360-379 189 189 189 333 333 333 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 186 186 186 330 330 330 F F F 21 420-439 189 189 189 333 333 333 F F F 22 440-459 188 188 188 332 332 332 F F F 23 460-479 188 188 188 331 331 331 F F F 24 480-499 188 188 188 331 331 331 F F F 25 500-519 184 184 184 327 327 327 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 189 189 189 333 333 333 F F F 29 580-599 189 189 189 333 333 333 F F F Max gen: 189, Max flush: 333 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.561752346Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.606802433Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.607407986Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.607453786Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.607469052Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.607483219Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944} {"msg":"Crucible 32b82e7e-55ef-4b39-8298-1a5d2e65be5d has session id: d563fa6a-96b2-4916-ae5d-32a3c32b8295","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608093311Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Upstairs opts: Upstairs UUID: 32b82e7e-55ef-4b39-8298-1a5d2e65be5d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608138221Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Crucible stats registered with UUID: 32b82e7e-55ef-4b39-8298-1a5d2e65be5d","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608162095Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:34.608182741Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608635108Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608662552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944} {"msg":"32b82e7e-55ef-4b39-8298-1a5d2e65be5d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608710501Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608727547Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"0","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608756571Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"0","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608771038Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"1","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608786234Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"1","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608800911Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"2","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608826355Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"2","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.60884979Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608873404Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608897739Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"ds_connection connected from Ok(127.0.0.1:38143)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608918824Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"ds_connection connected from Ok(127.0.0.1:43760)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.608974861Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"ds_connection connected from Ok(127.0.0.1:33181)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.609008304Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.609667864Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"1","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.609806973Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"0","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.610188866Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"1","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.610225378Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"2","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.610242064Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"0","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.610829391Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.610856855Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"2","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.610871882Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[0]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 331, 314, 333, 333, 330, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611383216Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[0]R generation[0..12]: [171, 182, 184, 184, 184, 188, 178, 189, 189, 186, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.61141024Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611426026Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[1]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 331, 314, 333, 333, 330, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611440513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[1]R generation[0..12]: [171, 182, 184, 184, 184, 188, 178, 189, 189, 186, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.61145395Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611467597Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[2]R flush_numbers[0..12]: [299, 323, 328, 327, 326, 331, 314, 333, 333, 330, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611481564Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[2]R generation[0..12]: [171, 182, 184, 184, 184, 188, 178, 189, 189, 186, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.61149507Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611508837Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-01-15T05:26:34.611532312Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611555797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611579911Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611602236Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"":"downstairs","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611625911Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611647816Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"0","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.61167389Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"1","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-15T05:26:34.611699424Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"client":"2","":"downstairs","session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611725728Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611749263Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"32b82e7e-55ef-4b39-8298-1a5d2e65be5d is now active with session: d563fa6a-96b2-4916-ae5d-32a3c32b8295","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.61176324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611775287Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944,"session_id":"d563fa6a-96b2-4916-ae5d-32a3c32b8295"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:34.611787544Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 36864 data: 16 19 20 17 20 15 15 15 13 02/30 Write block 270 len 36864 data: 12 12 11 11 11 9 8 9 16 03/30 Flush 04/30 Write block 467 len 24576 data: 14 15 13 11 11 13 05/30 Write block 120 len 16384 data: 13 9 10 11 06/30 Read block 11 len 8192 07/30 Flush 08/30 Write block 308 len 32768 data: 12 12 11 11 10 10 11 13 09/30 Write block 173 len 16384 data: 18 18 18 19 10/30 Read block 159 len 20480 11/30 Write block 4 len 40960 data: 11 11 11 11 14 12 10 9 10 8 12/30 Read block 320 len 16384 13/30 Flush 14/30 Write block 324 len 40960 data: 15 11 10 10 11 11 11 11 12 12 15/30 Read block 199 len 32768 16/30 Read block 460 len 28672 17/30 Read block 245 len 32768 18/30 Read block 497 len 4096 19/30 Write block 584 len 4096 data: 18 20/30 Write block 320 len 32768 data: 17 21 19 16 16 12 11 11 21/30 Write block 177 len 24576 data: 22 18 17 17 18 15 22/30 Read block 135 len 20480 23/30 Flush 24/30 Write block 354 len 36864 data: 17 16 20 21 18 12 13 15 15 25/30 Read block 228 len 16384 26/30 Read block 7 len 12288 27/30 Read block 592 len 32768 28/30 Read block 250 len 28672 29/30 Write block 14 len 32768 data: 8 7 9 7 7 10 11 13 30/30 Read block 475 len 32768 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 7 6 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:34.632415682Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1944} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 336 336 336 T T T 1 020-039 182 182 182 323 323 323 T T T 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 190 190 190 337 337 337 F F F 9 180-199 190 190 190 337 337 337 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 190 190 190 334 334 334 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 190 190 190 336 336 336 F F F 16 320-339 190 190 190 337 337 337 F F F 17 340-359 186 186 186 330 330 330 T T T 18 360-379 189 189 189 333 333 333 T T T 19 380-399 180 180 180 320 320 320 F F F 20 400-419 190 190 190 334 334 334 F F F 21 420-439 189 189 189 333 333 333 F F F 22 440-459 188 188 188 332 332 332 F F F 23 460-479 190 190 190 335 335 335 F F F 24 480-499 188 188 188 331 331 331 F F F 25 500-519 184 184 184 327 327 327 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 189 189 189 333 333 333 F F F 29 580-599 190 190 190 337 337 337 F F F Max gen: 190, Max flush: 337 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 190 190 190 337 337 337 F F F 9 180-199 190 190 190 337 337 337 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 190 190 190 334 334 334 F F F 14 280-299 189 189 189 333 333 333 F F F 15 300-319 190 190 190 336 336 336 F F F 16 320-339 190 190 190 337 337 337 F F F 17 340-359 191 191 191 338 338 338 F F F 18 360-379 191 191 191 338 338 338 F F F 19 380-399 180 180 180 320 320 320 F F F 20 400-419 190 190 190 334 334 334 F F F 21 420-439 189 189 189 333 333 333 F F F 22 440-459 188 188 188 332 332 332 F F F 23 460-479 190 190 190 335 335 335 F F F 24 480-499 188 188 188 331 331 331 F F F 25 500-519 184 184 184 327 327 327 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 189 189 189 333 333 333 F F F 29 580-599 190 190 190 337 337 337 F F F Max gen: 191, Max flush: 338 Begin loop 96 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.868169563Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.913374065Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.914101121Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.91414742Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.914162497Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.914176793Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952} {"msg":"Crucible 603766af-f488-4b29-adc8-a97b5bff9292 has session id: 761471a6-e8e2-4dae-81c7-8cb802236bb9","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.914773018Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Upstairs opts: Upstairs UUID: 603766af-f488-4b29-adc8-a97b5bff9292, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.914823017Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Crucible stats registered with UUID: 603766af-f488-4b29-adc8-a97b5bff9292","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.91485495Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:38.914877245Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915393928Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915421062Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952} {"msg":"603766af-f488-4b29-adc8-a97b5bff9292 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915490576Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915508422Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"0","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915531457Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"0","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915545994Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"1","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.91556019Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"1","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915573767Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"2","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915587964Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"2","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915600971Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915616887Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915640662Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34843)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915711146Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42492)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915758565Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55568)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.915785519Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.916628368Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"1","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.91666532Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"0","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.91732905Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"1","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.917369201Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"0","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.917510309Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"2","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.917907139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.917932793Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918146725Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"2","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[0]R flush_numbers[0..12]: [338, 338, 328, 327, 326, 331, 335, 333, 337, 337, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918636363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[0]R generation[0..12]: [191, 191, 184, 184, 184, 188, 190, 189, 190, 190, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918660688Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918675645Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[1]R flush_numbers[0..12]: [338, 338, 328, 327, 326, 331, 335, 333, 337, 337, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918689451Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[1]R generation[0..12]: [191, 191, 184, 184, 184, 188, 190, 189, 190, 190, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918708187Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918722774Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[2]R flush_numbers[0..12]: [338, 338, 328, 327, 326, 331, 335, 333, 337, 337, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918736281Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[2]R generation[0..12]: [191, 191, 184, 184, 184, 188, 190, 189, 190, 190, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918749938Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918764345Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918777012Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918790019Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918802836Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918822412Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"":"downstairs","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918835578Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918847316Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"0","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918860503Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"1","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.91887355Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"client":"2","":"downstairs","session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918886577Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918898604Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"603766af-f488-4b29-adc8-a97b5bff9292 is now active with session: 761471a6-e8e2-4dae-81c7-8cb802236bb9","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918914021Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918926118Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952,"session_id":"761471a6-e8e2-4dae-81c7-8cb802236bb9"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:38.918938015Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 281 len 24576 data: 20 18 13 11 10 12 02/30 Read block 317 len 32768 03/30 Write block 435 len 16384 data: 14 13 12 16 04/30 Read block 210 len 36864 05/30 Read block 405 len 36864 06/30 Write block 421 len 40960 data: 16 15 17 17 17 17 15 13 15 12 07/30 Read block 292 len 28672 08/30 Read block 134 len 16384 09/30 Read block 583 len 32768 10/30 Write block 276 len 40960 data: 9 10 17 18 18 21 19 14 12 11 11/30 Flush 12/30 Write block 338 len 24576 data: 18 16 15 12 10 9 13/30 Write block 453 len 12288 data: 11 13 10 14/30 Flush 15/30 Write block 489 len 28672 data: 14 11 9 8 7 11 11 16/30 Flush 17/30 Write block 342 len 4096 data: 11 18/30 Read block 525 len 12288 19/30 Read block 315 len 28672 20/30 Read block 343 len 24576 21/30 Read block 432 len 24576 22/30 Write block 357 len 4096 data: 22 23/30 Write block 441 len 24576 data: 15 14 15 17 17 15 24/30 Read block 185 len 40960 25/30 Write block 393 len 12288 data: 11 10 10 26/30 Write block 500 len 24576 data: 19 20 19 16 14 13 27/30 Write block 416 len 28672 data: 16 17 15 15 15 17 16 28/30 Read block 231 len 16384 29/30 Read block 559 len 24576 30/30 Read block 118 len 4096 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 1 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 14 13 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:38.939230499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1952} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 190 190 190 337 337 337 F F F 9 180-199 190 190 190 337 337 337 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 192 192 192 339 339 339 F F F 14 280-299 192 192 192 339 339 339 F F F 15 300-319 190 190 190 336 336 336 F F F 16 320-339 192 192 192 340 340 340 F F F 17 340-359 192 192 192 340 340 340 T T T 18 360-379 191 191 191 338 338 338 F F F 19 380-399 180 180 180 320 320 320 T T T 20 400-419 190 190 190 334 334 334 T T T 21 420-439 192 192 192 339 339 339 T T T 22 440-459 192 192 192 340 340 340 T T T 23 460-479 190 190 190 335 335 335 F F F 24 480-499 192 192 192 341 341 341 F F F 25 500-519 184 184 184 327 327 327 T T T 26 520-539 182 182 182 322 322 322 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 189 189 189 333 333 333 F F F 29 580-599 190 190 190 337 337 337 F F F Max gen: 192, Max flush: 341 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 190 190 190 337 337 337 F F F 9 180-199 190 190 190 337 337 337 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 192 192 192 339 339 339 F F F 14 280-299 192 192 192 339 339 339 F F F 15 300-319 190 190 190 336 336 336 F F F 16 320-339 192 192 192 340 340 340 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 191 191 191 338 338 338 F F F 19 380-399 193 193 193 342 342 342 F F F 20 400-419 193 193 193 342 342 342 F F F 21 420-439 193 193 193 342 342 342 F F F 22 440-459 193 193 193 342 342 342 F F F 23 460-479 190 190 190 335 335 335 F F F 24 480-499 192 192 192 341 341 341 F F F 25 500-519 193 193 193 342 342 342 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 189 189 189 333 333 333 F F F 29 580-599 190 190 190 337 337 337 F F F Max gen: 193, Max flush: 342 Begin loop 97 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.184204431Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.230738363Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.231377458Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.231426207Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.231441533Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.2314563Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960} {"msg":"Crucible 3eb369c2-da8c-49f5-b2a2-324edaaf1b31 has session id: c68b96e5-008b-4cfb-9497-692de920898b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232214128Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Upstairs opts: Upstairs UUID: 3eb369c2-da8c-49f5-b2a2-324edaaf1b31, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232257868Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Crucible stats registered with UUID: 3eb369c2-da8c-49f5-b2a2-324edaaf1b31","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232275244Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:43.232288492Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.23269383Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232719394Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960} {"msg":"3eb369c2-da8c-49f5-b2a2-324edaaf1b31 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232793707Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232812323Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"0","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232836217Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"0","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232860522Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"1","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232886696Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"1","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232918029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"2","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232947612Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"2","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.232975426Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.23300281Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.233031203Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33114)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.233060427Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"ds_connection connected from Ok(127.0.0.1:39687)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.233162414Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38251)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.233204154Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.233724156Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"0","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.233763797Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"1","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.23379443Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"2","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.234321401Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"1","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.23445966Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"0","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.234483664Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"2","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.234865008Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.234969364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[0]R flush_numbers[0..12]: [338, 338, 328, 327, 326, 331, 335, 333, 337, 337, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235090777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[0]R generation[0..12]: [191, 191, 184, 184, 184, 188, 190, 189, 190, 190, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235109982Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235124079Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[1]R flush_numbers[0..12]: [338, 338, 328, 327, 326, 331, 335, 333, 337, 337, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235137736Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[1]R generation[0..12]: [191, 191, 184, 184, 184, 188, 190, 189, 190, 190, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235151103Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.23516455Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[2]R flush_numbers[0..12]: [338, 338, 328, 327, 326, 331, 335, 333, 337, 337, 319, 329]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235177957Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[2]R generation[0..12]: [191, 191, 184, 184, 184, 188, 190, 189, 190, 190, 179, 185]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235191224Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235204801Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235217488Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235230465Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Next flush: 343","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235249931Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid"Wait for a query_work_queue command to finish before sending IO :1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235280734Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"":"downstairs","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235307138Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235330472Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"0","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235356387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"1","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235382211Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"client":"2","":"downstairs","session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235408565Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235433539Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"3eb369c2-da8c-49f5-b2a2-324edaaf1b31 is now active with session: c68b96e5-008b-4cfb-9497-692de920898b","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235459103Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235487297Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960,"session_id":"c68b96e5-008b-4cfb-9497-692de920898b"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:43.235512651Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 176 len 32768 data: 20 23 19 18 18 19 16 16 02/30 Read block 500 len 36864 03/30 Read block 194 len 8192 04/30 Flush 05/30 Flush 06/30 Write block 316 len 16384 data: 15 13 13 15 07/30 Read block 524 len 40960 08/30 Write block 305 len 40960 data: 8 11 11 13 13 12 12 11 11 12 09/30 Read block 471 len 36864 10/30 Read block 59 len 20480 11/30 Write block 393 len 28672 data: 12 11 11 12 13 15 13 12/30 Read block 546 len 12288 13/30 Read block 297 len 4096 14/30 Read block 304 len 8192 15/30 Write block 80 len 24576 data: 13 18 20 20 19 14 16/30 Write block 363 len 24576 data: 14 18 18 13 15 16 17/30 Write block 400 len 40960 data: 14 17 20 21 18 21 16 16 16 14 18/30 Write block 578 len 4096 data: 17 19/30 Write block 441 len 40960 data: 16 15 16 18 18 16 16 15 16 17 20/30 Write block 504 len 12288 data: 15 14 12 21/30 Write block 229 len 28672 data: 21 20 20 18 20 20 19 22/30 Write block 499 len 32768 data: 17 20 21 20 17 16 15 13 23/30 Write block 284 len 28672 data: 13 12 13 11 10 13 13 24/30 Read block 153 len 28672 25/30 Read block 316 len 12288 26/30 Write block 1 len 12288 data: 8 7 10 27/30 Read block 510 len 28672 28/30 Read block 278 len 20480 29/30 Read block 277 len 36864 30/30 Read block 200 len 32768 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 4 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 24 25 25 74 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:43.255001777Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1960} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 338 338 338 T T T 1 020-039 191 191 191 338 338 338 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 184 184 184 326 326 326 T T T 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 194 194 194 343 343 343 F F F 9 180-199 194 194 194 343 343 343 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 185 185 185 329 329 329 T T T 12 240-259 188 188 188 332 332 332 F F F 13 260-279 192 192 192 339 339 339 F F F 14 280-299 192 192 192 339 339 339 T T T 15 300-319 190 190 190 336 336 336 T T T 16 320-339 192 192 192 340 340 340 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 191 191 191 338 338 338 T T T 19 380-399 193 193 193 342 342 342 T T T 20 400-419 193 193 193 342 342 342 T T T 21 420-439 193 193 193 342 342 342 F F F 22 440-459 193 193 193 342 342 342 T T T 23 460-479 190 190 190 335 335 335 F F F 24 480-499 192 192 192 341 341 341 T T T 25 500-519 193 193 193 342 342 342 T T T 26 520-539 182 182 182 322 322 322 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 189 189 189 333 333 333 T T T 29 580-599 190 190 190 337 337 337 F F F Max gen: 194, Max flush: 343 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 344 344 344 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 195 195 195 344 344 344 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 194 194 194 343 343 343 F F F 9 180-199 194 194 194 343 343 343 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 195 195 195 344 344 344 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 192 192 192 339 339 339 F F F 14 280-299 195 195 195 344 344 344 F F F 15 300-319 195 195 195 344 344 344 F F F 16 320-339 192 192 192 340 340 340 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 195 195 195 344 344 344 F F F 19 380-399 195 195 195 344 344 344 F F F 20 400-419 195 195 195 344 344 344 F F F 21 420-439 193 193 193 342 342 342 F F F 22 440-459 195 195 195 344 344 344 F F F 23 460-479 190 190 190 335 335 335 F F F 24 480-499 195 195 195 344 344 344 F F F 25 500-519 195 195 195 344 344 344 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 188 188 188 331 331 331 F F F 28 560-579 195 195 195 344 344 344 F F F 29 580-599 190 190 190 337 337 337 F F F Max gen: 195, Max flush: 344 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.543619986Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:47.587141079Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.587506546Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.620158411Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.620975806Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.621018297Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.621035173Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.621053029Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968} {"msg":"Crucible 32a61d77-a19d-4391-bab0-00ef5583c596 has session id: 7ab82585-88f7-4d03-8fb5-2c2e461016c4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.621736144Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Upstairs opts: Upstairs UUID: 32a61d77-a19d-4391-bab0-00ef5583c596, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.621772126Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Crucible stats registered with UUID: 32a61d77-a19d-4391-bab0-00ef5583c596","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.621787493Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:47.62179992Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622216475Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622249788Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968} {"msg":"32a61d77-a19d-4391-bab0-00ef5583c596 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622303276Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622333489Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"0","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.6223712Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"0","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622398324Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"1","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622426098Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"1","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622452512Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"2","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622481175Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"2","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.62250617Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622529034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622557568Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:37139)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622583442Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38725)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.622608576Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36864)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.62263597Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.623204981Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"2","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.623232725Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"1","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.623455035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"0","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.623986255Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"2","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624018077Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"1","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624044741Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"0","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624567303Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624666141Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[0]R flush_numbers[0..12]: [344, 338, 328, 327, 344, 331, 335, 333, 343, 343, 319, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624784374Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[0]R generation[0..12]: [195, 191, 184, 184, 195, 188, 190, 189, 194, 194, 179, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624804709Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624822825Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[1]R flush_numbers[0..12]: [344, 338, 328, 327, 344, 331, 335, 333, 343, 343, 319, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624837132Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[1]R generation[0..12]: [195, 191, 184, 184, 195, 188, 190, 189, 194, 194, 179, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624850609Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624864126Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[2]R flush_numbers[0..12]: [344, 338, 328, 327, 344, 331, 335, 333, 343, 343, 319, 344]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624877413Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[2]R generation[0..12]: [195, 191, 184, 184, 195, 188, 190, 189, 194, 194, 179, 195]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.62489062Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624904587Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624917364Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624930291Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-15T05:26:47.624942898Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624962713Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"":"downstairs","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624975351Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624986958Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"0","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.624999715Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"1","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.625012322Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"client":"2","":"downstairs","session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.625027499Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.625039596Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"32a61d77-a19d-4391-bab0-00ef5583c596 is now active with session: 7ab82585-88f7-4d03-8fb5-2c2e461016c4","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.625051953Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.62506432Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968,"session_id":"7ab82585-88f7-4d03-8fb5-2c2e461016c4"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:47.625076158Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 8192 data: 15 16 02/30 Read block 557 len 24576 03/30 Read block 433 len 40960 04/30 Write block 400 len 16384 data: 15 18 21 22 05/30 Read block 84 len 4096 06/30 Read block 172 len 20480 07/30 Write block 279 len 28672 data: 19 19 22 20 15 14 13 08/30 Read block 308 len 24576 09/30 Read block 505 len 12288 10/30 Read block 3 len 4096 11/30 Read block 160 len 8192 12/30 Read block 321 len 20480 13/30 Write block 227 len 40960 data: 22 21 22 21 21 19 21 21 20 17 14/30 Write block 228 len 32768 data: 22 23 22 22 20 22 22 21 15/30 Read block 46 len 4096 16/30 Read block 396 len 24576 17/30 Read block 515 len 40960 18/30 Write block 479 len 20480 data: 17 15 15 13 10 19/30 Read block 556 len 36864 20/30 Write block 542 len 20480 data: 14 15 17 18 17 21/30 Read block 213 len 24576 22/30 Read block 557 len 20480 23/30 Read block 429 len 20480 24/30 Write block 491 len 36864 data: 10 9 8 12 12 11 12 14 18 25/30 Flush 26/30 Write block 457 len 24576 data: 9 11 14 12 13 13 27/30 Write block 393 len 28672 data: 13 12 12 13 14 16 14 28/30 Read block 420 len 40960 29/30 Write block 280 len 32768 data: 20 23 21 16 15 14 14 12 30/30 Write block 2 len 20480 data: 8 11 12 12 12 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:47.645769831Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1968} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 344 344 344 T T T 1 020-039 191 191 191 338 338 338 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 195 195 195 344 344 344 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 196 196 196 345 345 345 F F F 9 180-199 194 194 194 343 343 343 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 196 196 196 345 345 345 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 196 196 196 345 345 345 F F F 14 280-299 196 196 196 345 345 345 T T T 15 300-319 195 195 195 344 344 344 F F F 16 320-339 192 192 192 340 340 340 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 195 195 195 344 344 344 F F F 19 380-399 195 195 195 344 344 344 T T T 20 400-419 196 196 196 345 345 345 F F F 21 420-439 193 193 193 342 342 342 F F F 22 440-459 195 195 195 344 344 344 T T T 23 460-479 196 196 196 345 345 345 T T T 24 480-499 196 196 196 345 345 345 F F F 25 500-519 195 195 195 344 344 344 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 196 196 196 345 345 345 F F F 28 560-579 195 195 195 344 344 344 F F F 29 580-599 190 190 190 337 337 337 F F F Max gen: 196, Max flush: 345 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 346 346 346 F F F 1 020-039 191 191 191 338 338 338 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 195 195 195 344 344 344 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 189 189 189 333 333 333 F F F 8 160-179 196 196 196 345 345 345 F F F 9 180-199 194 194 194 343 343 343 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 196 196 196 345 345 345 F F F 12 240-259 188 188 188 332 332 332 F F F 13 260-279 196 196 196 345 345 345 F F F 14 280-299 197 197 197 346 346 346 F F F 15 300-319 195 195 195 344 344 344 F F F 16 320-339 192 192 192 340 340 340 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 195 195 195 344 344 344 F F F 19 380-399 197 197 197 346 346 346 F F F 20 400-419 196 196 196 345 345 345 F F F 21 420-439 193 193 193 342 342 342 F F F 22 440-459 197 197 197 346 346 346 F F F 23 460-479 197 197 197 346 346 346 F F F 24 480-499 196 196 196 345 345 345 F F F 25 500-519 195 195 195 344 344 344 F F F 26 520-539 182 182 182 322 322 322 F F F 27 540-559 196 196 196 345 345 345 F F F 28 560-579 195 195 195 344 344 344 F F F 29 580-599 190 190 190 337 337 337 F F F Max gen: 197, Max flush: 346 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.924152348Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.970212326Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.970919776Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"008b9b7890522309c5657fa001c5655325f06c1f\",\n git_commit_timestamp: \"2025-01-15T05:03:03.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.970965076Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.970980022Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.970994389Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976} {"msg":"Crucible a7ca7d83-5595-4c01-a252-7c90aa47dc85 has session id: 83d3c5e9-519c-4b54-b077-13b89c79a6f7","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.971579516Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Upstairs opts: Upstairs UUID: a7ca7d83-5595-4c01-a252-7c90aa47dc85, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.971623337Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Crucible stats registered with UUID: a7ca7d83-5595-4c01-a252-7c90aa47dc85","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.971647501Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:51.971668207Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972170213Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972205035Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976} {"msg":"a7ca7d83-5595-4c01-a252-7c90aa47dc85 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972275599Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972303233Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"0","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972341004Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"0","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972368558Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"1","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972399541Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"1","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972431574Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"2","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972460387Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"2","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972487691Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972512325Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.9725371Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55158)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972568123Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61105)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.972597726Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36947)","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.97275661Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.973263845Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"2","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.973291449Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"0","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.973402034Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"1","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.973823598Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"0","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.97390294Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"1","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.973988971Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"2","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974464483Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974499705Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[0]R flush_numbers[0..12]: [346, 338, 328, 327, 344, 331, 335, 333, 345, 343, 319, 345]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974557452Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[0]R generation[0..12]: [197, 191, 184, 184, 195, 188, 190, 189, 196, 194, 179, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974585056Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974600842Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[1]R flush_numbers[0..12]: [346, 338, 328, 327, 344, 331, 335, 333, 345, 343, 319, 345]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974615139Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[1]R generation[0..12]: [197, 191, 184, 184, 195, 188, 190, 189, 196, 194, 179, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974628766Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974642513Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[2]R flush_numbers[0..12]: [346, 338, 328, 327, 344, 331, 335, 333, 345, 343, 319, 345]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.97465635Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[2]R generation[0..12]: [197, 191, 184, 184, 195, 188, 190, 189, 196, 194, 179, 196]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974670236Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974684403Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-15T05:26:51.97469705Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974716626Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Next flush: 347","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974739831Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974763615Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"":"downstairs","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974778552Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974790559Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"0","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974803536Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"1","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974823072Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"client":"2","":"downstairs","session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974846187Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974859464Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"a7ca7d83-5595-4c01-a252-7c90aa47dc85 is now active with session: 83d3c5e9-519c-4b54-b077-13b89c79a6f7","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974871941Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974883658Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976,"session_id":"83d3c5e9-519c-4b54-b077-13b89c79a6f7"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-01-15T05:26:51.974899105Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 529 len 4096 data: 15 02/30 Write block 392 len 24576 data: 13 14 13 13 14 15 03/30 Write block 451 len 16384 data: 13 13 12 14 04/30 Write block 317 len 8192 data: 14 14 05/30 Write block 428 len 28672 data: 14 16 13 13 16 14 15 06/30 Write block 21 len 4096 data: 14 07/30 Write block 388 len 36864 data: 16 15 16 13 14 15 14 14 15 08/30 Read block 570 len 16384 09/30 Write block 407 len 36864 data: 17 17 15 16 19 19 20 17 18 10/30 Write block 446 len 40960 data: 17 17 16 17 18 14 14 13 15 11 11/30 Write block 259 len 36864 data: 17 16 16 12 14 16 16 18 17 12/30 Flush 13/30 Write block 158 len 32768 data: 18 19 18 17 18 20 15 16 14/30 Flush 15/30 Flush 16/30 Write block 368 len 28672 data: 17 13 14 13 12 15 12 17/30 Read block 475 len 8192 18/30 Read block 482 len 40960 19/30 Read block 378 len 12288 20/30 Read block 421 len 20480 21/30 Read block 341 len 20480 22/30 Write block 336 len 8192 data: 19 18 23/30 Write block 183 len 12288 data: 17 14 12 24/30 Write block 388 len 12288 data: 17 16 17 25/30 Write block 576 len 28672 data: 17 17 18 19 18 21 21 26/30 Read block 449 len 36864 27/30 Read block 333 len 24576 28/30 Write block 293 len 4096 data: 12 29/30 Read block 577 len 24576 30/30 Write block 556 len 40960 data: 10 11 9 8 10 7 10 11 10 8 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-15T05:26:51.993270674Z","hostname":"w-01JHM6ARD39SFM9SXVPVHKAJ0E","pid":1976} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 346 346 346 F F F 1 020-039 198 198 198 347 347 347 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 195 195 195 344 344 344 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 198 198 198 348 348 348 F F F 8 160-179 198 198 198 348 348 348 F F F 9 180-199 194 194 194 343 343 343 T T T 10 200-219 179 179 179 319 319 319 F F F 11 220-239 196 196 196 345 345 345 F F F 12 240-259 198 198 198 347 347 347 F F F 13 260-279 198 198 198 347 347 347 F F F 14 280-299 197 197 197 346 346 346 T T T 15 300-319 198 198 198 347 347 347 F F F 16 320-339 192 192 192 340 340 340 T T T 17 340-359 193 193 193 342 342 342 F F F 18 360-379 195 195 195 344 344 344 T T T 19 380-399 198 198 198 347 347 347 T T T 20 400-419 198 198 198 347 347 347 F F F 21 420-439 198 198 198 347 347 347 F F F 22 440-459 198 198 198 347 347 347 F F F 23 460-479 197 197 197 346 346 346 F F F 24 480-499 196 196 196 345 345 345 F F F 25 500-519 195 195 195 344 344 344 F F F 26 520-539 198 198 198 347 347 347 F F F 27 540-559 196 196 196 345 345 345 T T T 28 560-579 195 195 195 344 344 344 T T T 29 580-599 190 190 190 337 337 337 T T T Max gen: 198, Max flush: 348 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 346 346 346 F F F 1 020-039 198 198 198 347 347 347 F F F 2 040-059 184 184 184 328 328 328 F F F 3 060-079 184 184 184 327 327 327 F F F 4 080-099 195 195 195 344 344 344 F F F 5 100-119 188 188 188 331 331 331 F F F 6 120-139 190 190 190 335 335 335 F F F 7 140-159 198 198 198 348 348 348 F F F 8 160-179 198 198 198 348 348 348 F F F 9 180-199 199 199 199 349 349 349 F F F 10 200-219 179 179 179 319 319 319 F F F 11 220-239 196 196 196 345 345 345 F F F 12 240-259 198 198 198 347 347 347 F F F 13 260-279 198 198 198 347 347 347 F F F 14 280-299 199 199 199 349 349 349 F F F 15 300-319 198 198 198 347 347 347 F F F 16 320-339 199 199 199 349 349 349 F F F 17 340-359 193 193 193 342 342 342 F F F 18 360-379 199 199 199 349 349 349 F F F 19 380-399 199 199 199 349 349 349 F F F 20 400-419 198 198 198 347 347 347 F F F 21 420-439 198 198 198 347 347 347 F F F 22 440-459 198 198 198 347 347 347 F F F 23 460-479 197 197 197 346 346 346 F F F 24 480-499 196 196 196 345 345 345 F F F 25 500-519 195 195 195 344 344 344 F F F 26 520-539 198 198 198 347 347 347 F F F 27 540-559 199 199 199 349 349 349 F F F 28 560-579 199 199 199 349 349 349 F F F 29 580-599 199 199 199 349 349 349 F F F Max gen: 199, Max flush: 349 8:05 Test duration Test completed real 8:04.747420145 user 49.460392526 sys 19.316405202 trap 0.069835999 tflt 0.086425697 dflt 0.132081118 kflt 0.006459820 lock 13:01:41.975242537 slp 39:32.359409962 lat 5.760950253 stop 0.062118577