Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.34889168 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.020054448 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.020534415 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1193 1194 1195 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.758761415Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.8189434Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.821007211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.821385557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.821407157Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.821425627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198} {"msg":"Crucible a3c6ffa7-1186-49f7-9da1-1b567693d670 has session id: c334611a-f7aa-4e05-8cd2-89c94dee486d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.822393428Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Upstairs opts: Upstairs UUID: a3c6ffa7-1186-49f7-9da1-1b567693d670, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.822516337Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Crucible stats registered with UUID: a3c6ffa7-1186-49f7-9da1-1b567693d670","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.822536827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:17.822563377Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824116673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824233622Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198} {"msg":"a3c6ffa7-1186-49f7-9da1-1b567693d670 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824272762Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824297021Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"0","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824808157Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"0","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824842106Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"1","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824867676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"1","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824889706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"2","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824914376Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"2","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824937356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824960865Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.824984215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61506)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.825027805Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"ds_connection connected from Ok(127.0.0.1:61180)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.825045284Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36640)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.825059414Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.826880308Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"1","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.827034086Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"2","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.827141165Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"0","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.828258915Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"2","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.828454974Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"1","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.828488343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"0","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.829482734Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.829596223Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.82996544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.82998665Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.83000123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.83001521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830029319Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830043309Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830056529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830069579Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830087979Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830100909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830113829Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830126529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830138998Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"":"downstairs","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830151878Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830163828Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"0","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830176768Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"1","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830189558Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"client":"2","":"downstairs","session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830202578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830214448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"a3c6ffa7-1186-49f7-9da1-1b567693d670 is now active with session: c334611a-f7aa-4e05-8cd2-89c94dee486d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830226948Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830238588Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198,"session_id":"c334611a-f7aa-4e05-8cd2-89c94dee486d"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830250297Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198} {"msg":"Activated sub_volume a3c6ffa7-1186-49f7-9da1-1b567693d670","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:17.830335247Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1198} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.948170109Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.993271401Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.994054914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.994100943Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.994115833Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.994130153Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200} {"msg":"Crucible a9f0bb56-428e-4545-abc2-305732e958b9 has session id: 04b2d44d-b568-42a8-b855-dfba649b0f2d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.994616189Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Upstairs opts: Upstairs UUID: a9f0bb56-428e-4545-abc2-305732e958b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.994735547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Crucible stats registered with UUID: a9f0bb56-428e-4545-abc2-305732e958b9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.994771627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:21.994799277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995220093Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995245133Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200} {"msg":"a9f0bb56-428e-4545-abc2-305732e958b9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995306042Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995323272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"0","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995353152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"0","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995367182Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"1","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995380971Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"1","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995393691Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"2","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995407441Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"2","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995420251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995433451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50957)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995449481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.99551646Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55494)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.99555819Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47314)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.995606989Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.996245614Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"1","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.996276633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"0","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.996439132Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"2","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.996938967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"1","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.996967737Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"0","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.996983267Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"2","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997531942Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997558032Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997615311Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997635161Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997649421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997662791Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997675741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997689421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.99770601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.9977197Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.99773356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.99774642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.99775936Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.99777167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.99778414Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"":"downstairs","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:41:21.99779691Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997813749Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"0","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997826769Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"1","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997839369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"client":"2","":"downstairs","session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997851959Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997863859Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"a9f0bb56-428e-4545-abc2-305732e958b9 is now active with session: 04b2d44d-b568-42a8-b855-dfba649b0f2d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997876409Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997887869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200,"session_id":"04b2d44d-b568-42a8-b855-dfba649b0f2d"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997899799Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200} {"msg":"Activated sub_volume a9f0bb56-428e-4545-abc2-305732e958b9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:21.997913028Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 257 len 8192 data: 2 2 02/30 Read block 563 len 8192 03/30 Write block 480 len 8192 data: 2 2 04/30 Read block 89 len 20480 05/30 Write block 240 len 24576 data: 2 2 2 2 2 2 06/30 Write block 10 len 16384 data: 2 2 2 2 07/30 Read block 467 len 20480 08/30 Read block 184 len 16384 09/30 Flush 10/30 Write block 4 len 16384 data: 2 2 2 2 11/30 Write block 379 len 16384 data: 2 2 2 2 12/30 Write block 304 len 12288 data: 2 2 2 13/30 Write block 371 len 24576 data: 2 2 2 2 2 2 14/30 Read block 567 len 24576 15/30 Read block 53 len 40960 16/30 Read block 398 len 20480 17/30 Read block 355 len 20480 18/30 Flush 19/30 Write block 168 len 32768 data: 2 2 2 2 2 2 2 2 20/30 Read block 238 len 20480 21/30 Read block 266 len 12288 22/30 Read block 14 len 4096 23/30 Read block 179 len 12288 24/30 Read block 129 len 36864 25/30 Read block 462 len 20480 26/30 Write block 442 len 12288 data: 2 2 2 27/30 Read block 189 len 32768 28/30 Write block 451 len 32768 data: 2 2 2 2 2 2 2 2 29/30 Read block 337 len 24576 30/30 Read block 162 len 4096 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 8 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:22.01878188Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1200} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 T T T 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 T T T 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 3 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 2 2 2 3 3 3 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.241171586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.286977311Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.287629395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.287673615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.287689525Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.287717984Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208} {"msg":"Crucible a88ff2af-d30b-48b1-a1aa-278d4b25d9d2 has session id: 9dbf6773-c784-4df0-9aa7-6e1898a5e515","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.288393058Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Upstairs opts: Upstairs UUID: a88ff2af-d30b-48b1-a1aa-278d4b25d9d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.288433328Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Crucible stats registered with UUID: a88ff2af-d30b-48b1-a1aa-278d4b25d9d2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.288450098Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:26.288462938Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.288873854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.288898814Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208} {"msg":"a88ff2af-d30b-48b1-a1aa-278d4b25d9d2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.288955393Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.288973013Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"0","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289001943Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"0","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289015582Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"1","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289029413Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"1","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289042532Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"2","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289064572Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"2","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289083332Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289097732Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289111152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"ds_connection connected from Ok(127.0.0.1:59411)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289175781Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"ds_connection connected from Ok(127.0.0.1:35528)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.289196011Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"ds_connection connected from Ok(127.0.0.1:44564)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.28929249Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.290005254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"2","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.290047043Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"1","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.290125272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"0","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.290795397Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"2","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.290824256Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"1","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.290840906Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"0","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291407201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291433981Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[0]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291463871Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[0]R generation[0..12]: [2, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.29148296Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.29149817Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[1]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.29151198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[1]R generation[0..12]: [2, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.29152511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.29153896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[2]R flush_numbers[0..12]: [3, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.29155215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[2]R generation[0..12]: [2, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291565199Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291578709Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291591349Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291604149Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291616579Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"All extents match","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2025-03-18T01:41:26.291635369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"":"downstairs","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291648319Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291659949Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"0","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291673009Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"1","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291687589Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"client":"2","":"downstairs","session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291716218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291729168Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"a88ff2af-d30b-48b1-a1aa-278d4b25d9d2 is now active with session: 9dbf6773-c784-4df0-9aa7-6e1898a5e515","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291741668Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291753268Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208,"session_id":"9dbf6773-c784-4df0-9aa7-6e1898a5e515"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291765148Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208} {"msg":"Activated sub_volume a88ff2af-d30b-48b1-a1aa-278d4b25d9d2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:26.291775848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 52 len 4096 data: 2 02/30 Write block 532 len 16384 data: 2 2 2 2 03/30 Read block 281 len 20480 04/30 Read block 495 len 24576 05/30 Write block 52 len 20480 data: 3 2 2 2 2 06/30 Write block 216 len 28672 data: 2 2 2 2 2 2 2 07/30 Flush 08/30 Write block 334 len 12288 data: 2 2 2 09/30 Write block 229 len 16384 data: 2 2 2 2 10/30 Read block 512 len 12288 11/30 Read block 154 len 36864 12/30 Write block 382 len 8192 data: 3 2 13/30 Read block 423 len 24576 14/30 Read block 188 len 20480 15/30 Read block 198 len 12288 16/30 Read block 135 len 24576 17/30 Flush 18/30 Write block 120 len 8192 data: 2 2 19/30 Read block 138 len 4096 20/30 Write block 89 len 24576 data: 2 2 2 2 2 2 21/30 Write block 101 len 24576 data: 2 2 2 2 2 2 22/30 Write block 15 len 24576 data: 2 2 2 2 2 2 23/30 Flush 24/30 Write block 492 len 16384 data: 2 2 2 2 25/30 Write block 272 len 12288 data: 2 2 2 26/30 Read block 458 len 16384 27/30 Write block 321 len 20480 data: 2 2 2 2 2 28/30 Write block 251 len 32768 data: 2 2 2 2 2 2 3 3 29/30 Read block 270 len 8192 30/30 Write block 148 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:26.309485447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 5 5 5 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 4 4 4 7 7 7 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 1 1 1 1 1 1 T T T 8 160-179 3 3 3 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 4 4 4 5 5 5 F F F 11 220-239 4 4 4 6 6 6 F F F 12 240-259 2 2 2 2 2 2 T T T 13 260-279 1 1 1 1 1 1 T T T 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 4 4 4 6 6 6 T T T 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 4 4 4 6 6 6 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 2 2 2 2 2 2 T T T 25 500-519 1 1 1 1 1 1 F F F 26 520-539 4 4 4 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 7 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 5 5 5 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 4 4 4 7 7 7 F F F 6 120-139 4 4 4 7 7 7 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 4 4 4 5 5 5 F F F 11 220-239 4 4 4 6 6 6 F F F 12 240-259 5 5 5 8 8 8 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 5 5 5 8 8 8 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 4 4 4 6 6 6 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 5 5 5 8 8 8 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 4 4 4 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 8 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.552876123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.597548259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.598233693Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.598306923Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.598339412Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.598360572Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216} {"msg":"Crucible 3d398fa2-fc16-411e-aad0-1e8faff18e3e has session id: 2f3b7c86-9123-41e8-b7b5-c71735115a32","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.598887737Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Upstairs opts: Upstairs UUID: 3d398fa2-fc16-411e-aad0-1e8faff18e3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.598924177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Crucible stats registered with UUID: 3d398fa2-fc16-411e-aad0-1e8faff18e3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.598941897Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:30.598958026Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599394573Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599423622Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216} {"msg":"3d398fa2-fc16-411e-aad0-1e8faff18e3e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599512832Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599531121Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"0","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599561521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"0","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599579931Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"1","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599596941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"1","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.59962134Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"2","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.59964883Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"2","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.59966745Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.5996912Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.5997193Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"ds_connection connected from Ok(127.0.0.1:58873)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599767219Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"ds_connection connected from Ok(127.0.0.1:42772)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.599819919Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"ds_connection connected from Ok(127.0.0.1:34013)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.600042147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.600448883Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"2","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.600488683Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"0","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.601168926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"0","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.601319325Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"1","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.601417474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"2","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.601807181Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.601946199Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"1","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602105828Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[0]R flush_numbers[0..12]: [7, 7, 5, 1, 7, 7, 7, 8, 4, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602542894Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[0]R generation[0..12]: [4, 4, 4, 1, 4, 4, 4, 5, 3, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602562034Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602576484Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[1]R flush_numbers[0..12]: [7, 7, 5, 1, 7, 7, 7, 8, 4, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602589974Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[1]R generation[0..12]: [4, 4, 4, 1, 4, 4, 4, 5, 3, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602602804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602616393Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[2]R flush_numbers[0..12]: [7, 7, 5, 1, 7, 7, 7, 8, 4, 1, 5, 6]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602629573Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[2]R generation[0..12]: [4, 4, 4, 1, 4, 4, 4, 5, 3, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602642433Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602655983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602668543Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602682033Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602705173Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602719523Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"":"downstairs","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602732572Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Transition from WaitQuorum to Active","v"Wait for a query_work_queue command to finish before sending IO :0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602754342Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"0","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602767472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"1","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602780322Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"client":"2","":"downstairs","session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602793192Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602805082Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"3d398fa2-fc16-411e-aad0-1e8faff18e3e is now active with session: 2f3b7c86-9123-41e8-b7b5-c71735115a32","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602817452Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602829062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216,"session_id":"2f3b7c86-9123-41e8-b7b5-c71735115a32"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602841062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216} {"msg":"Activated sub_volume 3d398fa2-fc16-411e-aad0-1e8faff18e3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:30.602851891Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 237 len 28672 data: 2 2 2 3 3 3 3 03/30 Read block 570 len 28672 04/30 Read block 372 len 36864 05/30 Write block 543 len 36864 data: 2 2 2 2 2 2 2 2 2 06/30 Read block 585 len 8192 07/30 Write block 428 len 40960 data: 2 2 2 2 2 2 2 2 2 2 08/30 Read block 430 len 40960 09/30 Flush 10/30 Write block 114 len 16384 data: 2 2 2 2 11/30 Write block 574 len 4096 data: 2 12/30 Read block 328 len 16384 13/30 Write block 203 len 20480 data: 2 2 2 2 2 14/30 Read block 463 len 8192 15/30 Read block 324 len 24576 16/30 Write block 521 len 4096 data: 2 17/30 Write block 335 len 32768 data: 3 3 2 2 2 2 2 2 18/30 Write block 334 len 4096 data: 3 19/30 Read block 189 len 36864 20/30 Write block 507 len 28672 data: 2 2 2 2 2 2 2 21/30 Read block 428 len 20480 22/30 Write block 519 len 36864 data: 2 2 3 2 2 2 2 2 2 23/30 Write block 14 len 8192 data: 2 3 24/30 Read block 549 len 24576 25/30 Flush 26/30 Write block 233 len 32768 data: 2 2 2 2 3 3 3 4 27/30 Read block 14 len 16384 28/30 Read block 536 len 12288 29/30 Write block 123 len 8192 data: 2 2 30/30 Write block 557 len 20480 data: 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:30.623496955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 11 11 11 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 5 5 5 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 4 4 4 7 7 7 T T T 7 140-159 5 5 5 8 8 8 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 6 6 6 10 10 10 T T T 12 240-259 6 6 6 10 10 10 T T T 13 260-279 5 5 5 8 8 8 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 6 6 6 11 11 11 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 4 4 4 6 6 6 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 6 6 6 10 10 10 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 5 5 5 8 8 8 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 6 6 6 10 10 10 T T T 28 560-579 6 6 6 11 11 11 T T T 29 580-599 1 1 1 1 1 1 F F F Max gen: 6, Max flush: 11 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 11 11 11 F F F 1 020-039 4 4 4 7 7 7 F F F 2 040-059 4 4 4 5 5 5 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 7 7 7 12 12 12 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 6 6 6 11 11 11 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 4 4 4 6 6 6 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 6 6 6 10 10 10 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 5 5 5 8 8 8 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 6 6 6 11 11 11 F F F 27 540-559 7 7 7 12 12 12 F F F 28 560-579 7 7 7 12 12 12 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 7, Max flush: 12 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.862876616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:34.906585541Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.906999467Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.938894509Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.939487633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.939528713Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.939546163Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.939563862Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224} {"msg":"Crucible 2271f3e7-e269-47f7-bfc0-e0862084562f has session id: 55865973-4e61-4834-a665-876c2a8ee7e3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940145017Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Upstairs opts: Upstairs UUID: 2271f3e7-e269-47f7-bfc0-e0862084562f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940177357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Crucible stats registered with UUID: 2271f3e7-e269-47f7-bfc0-e0862084562f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940197927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:34.940211597Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940628183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940653273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224} {"msg":"2271f3e7-e269-47f7-bfc0-e0862084562f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940711792Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940729552Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"0","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940759021Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"0","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940772472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"1","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940786231Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"1","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940798991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"2","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940812681Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"2","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940825641Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940839111Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.940852461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39972)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.94096902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49751)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.941007779Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:42394)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.941131548Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.941807252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"2","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.941846932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"0","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.941877482Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"1","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.942425557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"2","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.942523196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"0","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.942552235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"1","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.942918312Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.94310034Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[0]R flush_numbers[0..12]: [11, 7, 5, 1, 7, 11, 12, 8, 4, 1, 11, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943236399Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[0]R generation[0..12]: [6, 4, 4, 1, 4, 6, 7, 5, 3, 1, 6, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943259169Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943274879Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[1]R flush_numbers[0..12]: [11, 7, 5, 1, 7, 11, 12, 8, 4, 1, 11, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943294539Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[1]R generation[0..12]: [6, 4, 4, 1, 4, 6, 7, 5, 3, 1, 6, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943319858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943351588Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[2]R flush_numbers[0..12]: [11, 7, 5, 1, 7, 11, 12, 8, 4, 1, 11, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943377638Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[2]R generation[0..12]: [6, 4, 4, 1, 4, 6, 7, 5, 3, 1, 6, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943399678Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943417308Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:41:34.943441207Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943465407Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943489617Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943503797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"":"downstairs","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943516757Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943532716Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"0","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943556506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"1","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943581306Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"client":"2","":"downstairs","session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943603116Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943615506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"2271f3e7-e269-47f7-bfc0-e0862084562f is now active with session: 55865973-4e61-4834-a665-876c2a8ee7e3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943631676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943656655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224,"session_id":"55865973-4e61-4834-a665-876c2a8ee7e3"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943679215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224} {"msg":"Activated sub_volume 2271f3e7-e269-47f7-bfc0-e0862084562f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:34.943700465Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 395 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Write block 86 len 28672 data: 2 2 2 3 3 3 3 03/30 Write block 471 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 16 len 32768 05/30 Write block 452 len 40960 data: 3 3 3 3 3 3 3 2 2 2 06/30 Write block 534 len 36864 data: 3 3 2 2 2 2 2 2 2 07/30 Read block 593 len 20480 08/30 Read block 290 len 40960 09/30 Write block 48 len 40960 data: 2 2 2 2 4 3 3 3 3 2 10/30 Read block 512 len 4096 11/30 Write block 41 len 4096 data: 2 12/30 Write block 60 len 24576 data: 2 2 2 2 2 2 13/30 Read block 40 len 20480 14/30 Write block 7 len 4096 data: 3 15/30 Write block 173 len 16384 data: 3 3 3 2 16/30 Read block 366 len 12288 17/30 Read block 421 len 32768 18/30 Read block 235 len 24576 19/30 Read block 279 len 24576 20/30 Flush 21/30 Write block 350 len 12288 data: 2 2 2 22/30 Write block 416 len 28672 data: 2 2 2 2 2 2 2 23/30 Read block 380 len 24576 24/30 Read block 222 len 28672 25/30 Read block 400 len 28672 26/30 Write block 39 len 8192 data: 2 2 27/30 Read block 290 len 24576 28/30 Read block 324 len 12288 29/30 Read block 552 len 28672 30/30 Read block 572 len 4096 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:34.96416527Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 4 4 4 7 7 7 T T T 2 040-059 8 8 8 13 13 13 T T T 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 8 8 8 13 13 13 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 7 7 7 12 12 12 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 6 6 6 11 11 11 T T T 18 360-379 2 2 2 3 3 3 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 8 8 8 13 13 13 T T T 21 420-439 6 6 6 10 10 10 T T T 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 5 5 5 8 8 8 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 8 8 8 13 13 13 F F F 27 540-559 8 8 8 13 13 13 F F F 28 560-579 7 7 7 12 12 12 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 8, Max flush: 13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 9 9 9 14 14 14 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 5 5 5 8 8 8 F F F 8 160-179 8 8 8 13 13 13 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 6 6 6 11 11 11 F F F 11 220-239 7 7 7 12 12 12 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 9 9 9 14 14 14 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 9 9 9 14 14 14 F F F 21 420-439 9 9 9 14 14 14 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 5 5 5 8 8 8 F F F 25 500-519 6 6 6 11 11 11 F F F 26 520-539 8 8 8 13 13 13 F F F 27 540-559 8 8 8 13 13 13 F F F 28 560-579 7 7 7 12 12 12 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 9, Max flush: 14 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.236225527Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:39.280321437Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.280698934Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.31313948Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.313877164Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.313915224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.313929253Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.313942793Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232} {"msg":"Crucible abf8152d-cb0a-4dd3-b0be-dc47248742d5 has session id: e957d192-6fc9-4cf4-a7a3-9e683fe5eca2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.314616257Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Upstairs opts: Upstairs UUID: abf8152d-cb0a-4dd3-b0be-dc47248742d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.314655727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Crucible stats registered with UUID: abf8152d-cb0a-4dd3-b0be-dc47248742d5","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.314677527Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:39.314691177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315130373Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315153592Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232} {"msg":"abf8152d-cb0a-4dd3-b0be-dc47248742d5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315218072Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315236361Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"0","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315271991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"0","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315297011Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"1","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315335961Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"1","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.31536717Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"2","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.31539359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"2","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.31542119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.31544811Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315471879Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50460)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315496999Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63166)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315523759Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"ds_connection connected from Ok(127.0.0.1:54712)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.315547759Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.316235363Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"1","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.316277502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"2","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.316380291Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"0","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.316921356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"1","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317089325Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"0","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317135344Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"2","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.31759283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317681819Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[0]R flush_numbers[0..12]: [13, 14, 14, 13, 13, 11, 12, 8, 13, 1, 11, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317802038Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[0]R generation[0..12]: [8, 9, 9, 8, 8, 6, 7, 5, 8, 1, 6, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317821928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317837118Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[1]R flush_numbers[0..12]: [13, 14, 14, 13, 13, 11, 12, 8, 13, 1, 11, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317850648Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[1]R generation[0..12]: [8, 9, 9, 8, 8, 6, 7, 5, 8, 1, 6, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317863928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317877568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[2]R flush_numbers[0..12]: [13, 14, 14, 13, 13, 11, 12, 8, 13, 1, 11, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317890708Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[2]R generation[0..12]: [8, 9, 9, 8, 8, 6, 7, 5, 8, 1, 6, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317903568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317917087Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317929807Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317942667Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317955157Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317979377Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"":"downstairs","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.317992287Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318004047Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"0","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318017076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"1","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318029706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"client":"2","":"downstairs","session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318042516Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318054426Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"abf8152d-cb0a-4dd3-b0be-dc47248742d5 is now active with session: e957d192-6fc9-4cf4-a7a3-9e683fe5eca2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318067026Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318078856Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232,"session_id":"e957d192-6fc9-4cf4-a7a3-9e683fe5eca2"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318090916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232} {"msg":"Activated sub_volume abf8152d-cb0a-4dd3-b0be-dc47248742d5","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:39.318101736Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 565 len 16384 data: 2 2 2 2 02/30 Write block 356 len 8192 data: 2 2 03/30 Write block 527 len 16384 data: 3 2 2 2 04/30 Write block 540 len 24576 data: 3 3 3 3 3 3 05/30 Read block 571 len 4096 06/30 Write block 418 len 12288 data: 3 3 3 07/30 Read block 14 len 12288 08/30 Flush 09/30 Write block 175 len 24576 data: 4 3 2 2 2 2 10/30 Write block 311 len 24576 data: 2 2 2 2 2 2 11/30 Read block 483 len 16384 12/30 Write block 428 len 12288 data: 3 3 3 13/30 Write block 553 len 12288 data: 2 2 2 14/30 Flush 15/30 Write block 302 len 12288 data: 2 2 3 16/30 Flush 17/30 Write block 152 len 40960 data: 3 3 3 3 3 2 2 2 2 2 18/30 Write block 578 len 16384 data: 2 2 2 2 19/30 Read block 505 len 36864 20/30 Read block 61 len 20480 21/30 Read block 321 len 4096 22/30 Read block 558 len 32768 23/30 Write block 228 len 32768 data: 2 3 3 3 3 3 3 3 24/30 Read block 546 len 32768 25/30 Read block 547 len 12288 26/30 Write block 218 len 20480 data: 3 3 3 3 3 27/30 Write block 283 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Write block 497 len 20480 data: 2 2 2 2 2 29/30 Write block 535 len 28672 data: 4 3 3 3 3 4 4 30/30 Read block 449 len 36864 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 13 14 13 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:39.336902185Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 9 9 9 14 14 14 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 5 5 5 8 8 8 T T T 8 160-179 10 10 10 16 16 16 T T T 9 180-199 10 10 10 16 16 16 F F F 10 200-219 6 6 6 11 11 11 T T T 11 220-239 7 7 7 12 12 12 T T T 12 240-259 7 7 7 12 12 12 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 1 1 1 1 1 1 T T T 15 300-319 10 10 10 17 17 17 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 5 5 5 8 8 8 T T T 25 500-519 6 6 6 11 11 11 T T T 26 520-539 10 10 10 15 15 15 T T T 27 540-559 10 10 10 16 16 16 T T T 28 560-579 10 10 10 15 15 15 T T T 29 580-599 1 1 1 1 1 1 T T T Max gen: 10, Max flush: 17 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 8 8 13 13 13 F F F 1 020-039 9 9 9 14 14 14 F F F 2 040-059 9 9 9 14 14 14 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 11 11 11 18 18 18 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 11 11 11 18 18 18 F F F 11 220-239 11 11 11 18 18 18 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 5 5 5 8 8 8 F F F 14 280-299 11 11 11 18 18 18 F F F 15 300-319 10 10 10 17 17 17 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 11 11 11 18 18 18 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 11 11 11 18 18 18 F F F 28 560-579 11 11 11 18 18 18 F F F 29 580-599 11 11 11 18 18 18 F F F Max gen: 11, Max flush: 18 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.643269061Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:43.686995586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.687406882Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.719420143Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.720227745Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.720285135Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.720303415Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.720320914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240} {"msg":"Crucible e2e2b5e4-de5f-41a2-8ece-93ed185231dd has session id: 7415b96e-24fd-46a4-b40a-a80fdac51c4f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.720932479Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Upstairs opts: Upstairs UUID: e2e2b5e4-de5f-41a2-8ece-93ed185231dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.720976198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Crucible stats registered with UUID: e2e2b5e4-de5f-41a2-8ece-93ed185231dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721006668Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:43.721029218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721470414Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721500034Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240} {"msg":"e2e2b5e4-de5f-41a2-8ece-93ed185231dd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721565293Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721583373Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"0","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721613872Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"0","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721627663Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"1","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721641482Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"1","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721661712Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"2","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721688692Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"2","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721710052Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721726222Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721739882Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42266)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721757141Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62628)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.721783721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56435)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.72184415Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.722541784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"1","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.722628843Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"2","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.722722173Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"0","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.723397647Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"2","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.723427226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"0","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.723443486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"1","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724003451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.7241375Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[0]R flush_numbers[0..12]: [13, 14, 14, 13, 13, 11, 12, 18, 18, 16, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724190789Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[0]R generation[0..12]: [8, 9, 9, 8, 8, 6, 7, 11, 11, 10, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724209829Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724225269Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[1]R flush_numbers[0..12]: [13, 14, 14, 13, 13, 11, 12, 18, 18, 16, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724238969Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[1]R generation[0..12]: [8, 9, 9, 8, 8, 6, 7, 11, 11, 10, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724256909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724271828Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[2]R flush_numbers[0..12]: [13, 14, 14, 13, 13, 11, 12, 18, 18, 16, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724285288Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[2]R generation[0..12]: [8, 9, 9, 8, 8, 6, 7, 11, 11, 10, 11, 11]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724298418Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724312668Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724325458Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724344738Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Next flush: 19","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:41:43.724367748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724385547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"":"downstairs","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724398417Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724410147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"0","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724426407Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"1","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724449847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"client":"2","":"downstairs","session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724465447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724478037Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"e2e2b5e4-de5f-41a2-8ece-93ed185231dd is now active with session: 7415b96e-24fd-46a4-b40a-a80fdac51c4f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724492986Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724505586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240,"session_id":"7415b96e-24fd-46a4-b40a-a80fdac51c4f"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724526596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240} {"msg":"Activated sub_volume e2e2b5e4-de5f-41a2-8ece-93ed185231dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:43.724544926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 217 len 32768 data: 3 4 4 4 4 4 2 2 02/30 Read block 149 len 32768 03/30 Read block 314 len 8192 04/30 Write block 69 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Read block 338 len 16384 06/30 Write block 11 len 4096 data: 3 07/30 Read block 412 len 40960 08/30 Flush 09/30 Write block 566 len 28672 data: 3 3 3 2 2 2 2 10/30 Flush 11/30 Write block 54 len 16384 data: 4 4 4 3 12/30 Read block 424 len 20480 13/30 Read block 575 len 12288 14/30 Read block 415 len 40960 15/30 Flush 16/30 Write block 576 len 20480 data: 2 2 3 3 3 17/30 Write block 559 len 28672 data: 3 3 3 2 2 2 3 18/30 Read block 563 len 28672 19/30 Write block 26 len 12288 data: 2 2 2 20/30 Write block 191 len 40960 data: 2 2 2 2 2 2 2 2 2 2 21/30 Write block 277 len 24576 data: 2 2 2 2 2 2 22/30 Read block 360 len 32768 23/30 Flush 24/30 Flush 25/30 Write block 112 len 12288 data: 2 2 3 26/30 Read block 313 len 36864 27/30 Write block 71 len 36864 data: 3 3 3 3 3 3 3 2 2 28/30 Write block 277 len 24576 data: 3 3 3 3 3 3 29/30 Read block 445 len 28672 30/30 Write block 279 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:43.745384997Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 19 19 19 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 12 12 12 21 21 21 F F F 3 060-079 12 12 12 19 19 19 T T T 4 080-099 8 8 8 13 13 13 F F F 5 100-119 6 6 6 11 11 11 T T T 6 120-139 7 7 7 12 12 12 F F F 7 140-159 11 11 11 18 18 18 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 12 12 12 22 22 22 F F F 10 200-219 12 12 12 22 22 22 F F F 11 220-239 12 12 12 19 19 19 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 12 12 12 22 22 22 T T T 14 280-299 12 12 12 22 22 22 T T T 15 300-319 10 10 10 17 17 17 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 11 11 11 18 18 18 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 12 12 12 22 22 22 F F F 29 580-599 12 12 12 22 22 22 F F F Max gen: 12, Max flush: 22 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 19 19 19 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 12 12 12 21 21 21 F F F 3 060-079 13 13 13 23 23 23 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 13 13 13 23 23 23 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 11 11 11 18 18 18 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 12 12 12 22 22 22 F F F 10 200-219 12 12 12 22 22 22 F F F 11 220-239 12 12 12 19 19 19 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 13 13 13 23 23 23 F F F 14 280-299 13 13 13 23 23 23 F F F 15 300-319 10 10 10 17 17 17 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 10 10 10 15 15 15 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 11 11 11 18 18 18 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 12 12 12 22 22 22 F F F 29 580-599 12 12 12 22 22 22 F F F Max gen: 13, Max flush: 23 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.005894579Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:48.050074199Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.050457905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.081506074Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.082100709Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.082139018Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.082154218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.082168468Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248} {"msg":"Crucible b68c76da-668d-4d33-a35a-dc7a4dd173f8 has session id: c47ebe38-b4d9-42f8-a616-85abf9d16557","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.082693613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Upstairs opts: Upstairs UUID: b68c76da-668d-4d33-a35a-dc7a4dd173f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.082732673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Crucible stats registered with UUID: b68c76da-668d-4d33-a35a-dc7a4dd173f8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.082750713Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:48.082763683Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083152969Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083177469Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248} {"msg":"b68c76da-668d-4d33-a35a-dc7a4dd173f8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083239349Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083257528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"0","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083285968Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"0","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083299798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"1","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083313958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"1","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083332608Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"2","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083356998Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"2","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083379407Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083402757Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083425797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"ds_connection connected from Ok(127.0.0.1:35552)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083449607Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"ds_connection connected from Ok(127.0.0.1:40560)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083474646Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"ds_connection connected from Ok(127.0.0.1:42436)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.083498196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.08420171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"2","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.08423754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"1","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.084265599Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"0","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.084724305Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"1","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.084838014Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"2","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.084980133Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"0","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085367379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085496718Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[0]R flush_numbers[0..12]: [19, 22, 21, 23, 13, 23, 12, 18, 18, 22, 22, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085643277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[0]R generation[0..12]: [12, 12, 12, 13, 8, 13, 7, 11, 11, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085665557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085682426Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[1]R flush_numbers[0..12]: [19, 22, 21, 23, 13, 23, 12, 18, 18, 22, 22, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085705836Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[1]R generation[0..12]: [12, 12, 12, 13, 8, 13, 7, 11, 11, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085741596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085766806Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[2]R flush_numbers[0..12]: [19, 22, 21, 23, 13, 23, 12, 18, 18, 22, 22, 19]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085791365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[2]R generation[0..12]: [12, 12, 12, 13, 8, 13, 7, 11, 11, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085816115Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085843365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085859025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085872335Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085892155Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085926074Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"":"downstairs","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085952114Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085967574Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"0","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085981754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"1","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.085997884Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"client":"2","":"downstairs","session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.086021274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.086045163Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"b68c76da-668d-4d33-a35a-dc7a4dd173f8 is now active with session: c47ebe38-b4d9-42f8-a616-85abf9d16557","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.086067593Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.086080163Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248,"session_id":"c47ebe38-b4d9-42f8-a616-85abf9d16557"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.086093813Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248} {"msg":"Activated sub_volume b68c76da-668d-4d33-a35a-dc7a4dd173f8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:48.086113323Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 416 len 32768 data: 3 3 4 4 4 3 3 2 02/30 Flush 03/30 Write block 207 len 32768 data: 3 2 2 2 2 2 2 2 04/30 Write block 1 len 8192 data: 2 2 05/30 Read block 159 len 8192 06/30 Read block 10 len 8192 07/30 Flush 08/30 Write block 447 len 28672 data: 2 2 2 2 3 4 4 09/30 Read block 300 len 4096 10/30 Read block 385 len 20480 11/30 Read block 39 len 24576 12/30 Read block 286 len 20480 13/30 Read block 336 len 36864 14/30 Flush 15/30 Write block 578 len 32768 data: 4 4 4 3 2 2 2 2 16/30 Write block 8 len 20480 data: 2 2 3 4 3 17/30 Read block 418 len 4096 18/30 Write block 595 len 16384 data: 2 2 2 2 19/30 Read block 395 len 8192 20/30 Read block 466 len 40960 21/30 Read block 121 len 20480 22/30 Read block 39 len 12288 23/30 Read block 436 len 36864 24/30 Write block 233 len 16384 data: 4 4 4 3 25/30 Flush 26/30 Write block 223 len 28672 data: 3 3 2 2 2 3 4 27/30 Read block 151 len 32768 28/30 Write block 73 len 8192 data: 4 4 29/30 Write block 491 len 36864 data: 2 3 3 3 3 2 3 3 3 30/30 Write block 136 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:48.106909105Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 12 12 12 21 21 21 F F F 3 060-079 13 13 13 23 23 23 T T T 4 080-099 8 8 8 13 13 13 F F F 5 100-119 13 13 13 23 23 23 F F F 6 120-139 7 7 7 12 12 12 T T T 7 140-159 11 11 11 18 18 18 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 12 12 12 22 22 22 F F F 10 200-219 14 14 14 25 25 25 F F F 11 220-239 14 14 14 27 27 27 T T T 12 240-259 7 7 7 12 12 12 F F F 13 260-279 13 13 13 23 23 23 F F F 14 280-299 13 13 13 23 23 23 F F F 15 300-319 10 10 10 17 17 17 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 14 14 14 24 24 24 F F F 21 420-439 14 14 14 24 24 24 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 11 11 11 18 18 18 T T T 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 14 14 14 27 27 27 F F F 29 580-599 14 14 14 27 27 27 F F F Max gen: 14, Max flush: 27 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 12 12 12 21 21 21 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 13 13 13 23 23 23 F F F 6 120-139 15 15 15 28 28 28 F F F 7 140-159 11 11 11 18 18 18 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 12 12 12 22 22 22 F F F 10 200-219 14 14 14 25 25 25 F F F 11 220-239 15 15 15 28 28 28 F F F 12 240-259 7 7 7 12 12 12 F F F 13 260-279 13 13 13 23 23 23 F F F 14 280-299 13 13 13 23 23 23 F F F 15 300-319 10 10 10 17 17 17 F F F 16 320-339 6 6 6 11 11 11 F F F 17 340-359 10 10 10 15 15 15 F F F 18 360-379 2 2 2 3 3 3 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 14 14 14 24 24 24 F F F 21 420-439 14 14 14 24 24 24 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 15 15 15 28 28 28 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 14 14 14 27 27 27 F F F 29 580-599 14 14 14 27 27 27 F F F Max gen: 15, Max flush: 28 Begin loop 8 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.370480448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.414739767Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.415383761Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.415436141Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.415454921Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.41547321Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256} {"msg":"Crucible 6f9c064f-b661-4268-a404-f0421e683a52 has session id: 4279ab39-1683-429d-b7e9-22b1eaf3491a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416094065Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Upstairs opts: Upstairs UUID: 6f9c064f-b661-4268-a404-f0421e683a52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416127575Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Crucible stats registered with UUID: 6f9c064f-b661-4268-a404-f0421e683a52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416142354Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:52.416155254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416677079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416710399Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256} {"msg":"6f9c064f-b661-4268-a404-f0421e683a52 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416747569Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416769759Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"0","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416809218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"0","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416832468Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"1","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416857108Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"1","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416881688Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"2","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416907428Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"2","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416931427Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416957337Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.416981357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45656)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.417006847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39721)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.417031076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48906)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.417246144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.4177786Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"0","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.417820379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"2","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.417872489Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"1","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.418447214Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"2","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.418592522Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"1","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.418615942Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"0","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.418998818Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419132057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[0]R flush_numbers[0..12]: [27, 22, 21, 28, 13, 23, 28, 18, 18, 22, 25, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419203517Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[0]R generation[0..12]: [14, 12, 12, 15, 8, 13, 15, 11, 11, 12, 14, 15]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419223396Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419238276Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[1]R flush_numbers[0..12]: [27, 22, 21, 28, 13, 23, 28, 18, 18, 22, 25, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419251686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[1]R generation[0..12]: [14, 12, 12, 15, 8, 13, 15, 11, 11, 12, 14, 15]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419264876Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419278876Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[2]R flush_numbers[0..12]: [27, 22, 21, 28, 13, 23, 28, 18, 18, 22, 25, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419292466Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[2]R generation[0..12]: [14, 12, 12, 15, 8, 13, 15, 11, 11, 12, 14, 15]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419305806Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419327866Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419347705Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419370195Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419395275Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419420115Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"":"downstairs","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419434525Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419450285Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"0","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:41:52.419473784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"1","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419498084Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"client":"2","":"downstairs","session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419522454Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419536474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"6f9c064f-b661-4268-a404-f0421e683a52 is now active with session: 4279ab39-1683-429d-b7e9-22b1eaf3491a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419549384Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419568963Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256,"session_id":"4279ab39-1683-429d-b7e9-22b1eaf3491a"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419591633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256} {"msg":"Activated sub_volume 6f9c064f-b661-4268-a404-f0421e683a52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:52.419612553Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 117 len 32768 data: 3 2 2 3 3 2 3 3 02/30 Flush 03/30 Write block 179 len 4096 data: 3 04/30 Write block 444 len 24576 data: 3 2 2 3 3 3 05/30 Write block 37 len 28672 data: 2 2 3 3 3 2 2 06/30 Read block 490 len 28672 07/30 Write block 80 len 8192 data: 2 2 08/30 Write block 462 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Read block 565 len 32768 10/30 Write block 47 len 36864 data: 2 3 3 3 3 5 4 5 5 11/30 Write block 251 len 40960 data: 3 3 3 3 3 3 4 4 2 2 12/30 Read block 307 len 12288 13/30 Flush 14/30 Write block 366 len 36864 data: 2 2 2 2 2 3 3 3 3 15/30 Read block 257 len 36864 16/30 Flush 17/30 Write block 477 len 40960 data: 3 3 2 3 3 2 2 2 2 2 18/30 Flush 19/30 Write block 137 len 24576 data: 3 2 2 2 2 2 20/30 Write block 227 len 36864 data: 3 4 5 4 4 4 5 5 5 21/30 Flush 22/30 Write block 40 len 20480 data: 4 4 3 3 2 23/30 Write block 337 len 28672 data: 3 3 3 3 3 3 2 24/30 Read block 515 len 40960 25/30 Flush 26/30 Write block 343 len 12288 data: 3 2 2 27/30 Read block 4 len 32768 28/30 Read block 105 len 20480 29/30 Read block 544 len 32768 30/30 Read block 191 len 8192 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:52.441126208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 16 16 16 34 34 34 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 16 16 16 29 29 29 F F F 6 120-139 16 16 16 33 33 33 F F F 7 140-159 16 16 16 33 33 33 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 12 12 12 22 22 22 F F F 10 200-219 14 14 14 25 25 25 F F F 11 220-239 16 16 16 33 33 33 F F F 12 240-259 16 16 16 30 30 30 F F F 13 260-279 16 16 16 30 30 30 F F F 14 280-299 13 13 13 23 23 23 F F F 15 300-319 10 10 10 17 17 17 F F F 16 320-339 16 16 16 34 34 34 F F F 17 340-359 16 16 16 34 34 34 T T T 18 360-379 16 16 16 31 31 31 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 14 14 14 24 24 24 F F F 21 420-439 14 14 14 24 24 24 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 16 16 16 32 32 32 F F F 24 480-499 16 16 16 32 32 32 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 14 14 14 27 27 27 F F F 29 580-599 14 14 14 27 27 27 F F F Max gen: 16, Max flush: 34 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 16 16 16 30 30 30 F F F 2 040-059 16 16 16 34 34 34 F F F 3 060-079 15 15 15 28 28 28 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 16 16 16 29 29 29 F F F 6 120-139 16 16 16 33 33 33 F F F 7 140-159 16 16 16 33 33 33 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 12 12 12 22 22 22 F F F 10 200-219 14 14 14 25 25 25 F F F 11 220-239 16 16 16 33 33 33 F F F 12 240-259 16 16 16 30 30 30 F F F 13 260-279 16 16 16 30 30 30 F F F 14 280-299 13 13 13 23 23 23 F F F 15 300-319 10 10 10 17 17 17 F F F 16 320-339 16 16 16 34 34 34 F F F 17 340-359 17 17 17 35 35 35 F F F 18 360-379 16 16 16 31 31 31 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 14 14 14 24 24 24 F F F 21 420-439 14 14 14 24 24 24 F F F 22 440-459 16 16 16 30 30 30 F F F 23 460-479 16 16 16 32 32 32 F F F 24 480-499 16 16 16 32 32 32 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 12 12 12 22 22 22 F F F 28 560-579 14 14 14 27 27 27 F F F 29 580-599 14 14 14 27 27 27 F F F Max gen: 17, Max flush: 35 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.65738971Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:56.700767557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.701137864Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.732855657Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.733420812Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.733450372Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.733463651Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.733477391Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264} {"msg":"Crucible 12c06934-5d98-4bfc-a32a-1db05413eb37 has session id: 677602cc-a818-4542-95e7-648fad5919ef","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734072926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Upstairs opts: Upstairs UUID: 12c06934-5d98-4bfc-a32a-1db05413eb37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734100566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Crucible stats registered with UUID: 12c06934-5d98-4bfc-a32a-1db05413eb37","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734114695Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:56.734127105Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734582791Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734615281Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264} {"msg":"12c06934-5d98-4bfc-a32a-1db05413eb37 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.73466925Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.73469659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"0","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.73472833Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"0","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.73474318Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"1","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.73475735Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"1","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734773349Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"2","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734796759Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"2","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734819749Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734847369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734871629Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:56839)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734933078Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:64755)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734964888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:63424)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.734993347Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.735669371Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"1","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.735698641Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"2","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.73583014Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"0","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.736306476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"1","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.736348145Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"2","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.736381285Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"0","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.736883601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737010039Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[0]R flush_numbers[0..12]: [27, 30, 34, 28, 30, 29, 33, 33, 30, 22, 25, 33]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737084999Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[0]R generation[0..12]: [14, 16, 16, 15, 16, 16, 16, 16, 16, 12, 14, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737105718Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737123908Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[1]R flush_numbers[0..12]: [27, 30, 34, 28, 30, 29, 33, 33, 30, 22, 25, 33]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737138748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[1]R generation[0..12]: [14, 16, 16, 15, 16, 16, 16, 16, 16, 12, 14, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737152388Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737166618Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[2]R flush_numbers[0..12]: [27, 30, 34, 28, 30, 29, 33, 33, 30, 22, 25, 33]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737180418Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[2]R generation[0..12]: [14, 16, 16, 15, 16, 16, 16, 16, 16, 12, 14, 16]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737193708Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737207198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737219797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737232507Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737244947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737257357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"":"downstairs","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737269947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737281497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"0","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737294357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"1","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737306986Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"client":"2","":"downstairs","session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737321666Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737333706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"12c06934-5d98-4bfc-a32a-1db05413eb37 is now active with session: 677602cc-a818-4542-95e7-648fad5919ef","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737346086Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} Wait for a query_work_queue command to finish before sending IO {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737363286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264,"session_id":"677602cc-a818-4542-95e7-648fad5919ef"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737375396Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264} {"msg":"Activated sub_volume 12c06934-5d98-4bfc-a32a-1db05413eb37","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:41:56.737386526Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 279 len 36864 data: 5 5 4 4 3 3 3 3 3 03/30 Write block 294 len 40960 data: 2 2 2 2 2 2 2 2 3 3 04/30 Read block 518 len 36864 05/30 Write block 426 len 24576 data: 2 2 4 4 4 3 06/30 Write block 214 len 4096 data: 3 07/30 Read block 177 len 20480 08/30 Read block 390 len 12288 09/30 Flush 10/30 Write block 216 len 36864 data: 3 4 5 5 5 5 5 4 4 11/30 Write block 584 len 20480 data: 3 3 2 2 2 12/30 Write block 357 len 32768 data: 3 2 2 2 2 2 2 2 13/30 Write block 400 len 28672 data: 3 3 3 3 3 2 2 14/30 Read block 415 len 40960 15/30 Read block 37 len 28672 16/30 Flush 17/30 Write block 113 len 4096 data: 3 18/30 Write block 60 len 36864 data: 3 3 3 3 3 3 2 2 2 19/30 Flush 20/30 Write block 484 len 16384 data: 3 3 3 2 21/30 Write block 40 len 24576 data: 5 5 4 4 3 2 22/30 Write block 447 len 24576 data: 4 4 4 3 4 5 23/30 Flush 24/30 Write block 445 len 8192 data: 3 3 25/30 Read block 39 len 40960 26/30 Read block 397 len 24576 27/30 Write block 546 len 20480 data: 3 3 3 3 3 28/30 Write block 36 len 8192 data: 2 3 29/30 Write block 587 len 4096 data: 3 30/30 Read block 595 len 20480 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:41:56.756205506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 16 16 16 30 30 30 T T T 2 040-059 18 18 18 40 40 40 F F F 3 060-079 18 18 18 39 39 39 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 18 18 18 39 39 39 F F F 6 120-139 16 16 16 33 33 33 F F F 7 140-159 16 16 16 33 33 33 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 12 12 12 22 22 22 F F F 10 200-219 18 18 18 38 38 38 F F F 11 220-239 18 18 18 38 38 38 F F F 12 240-259 16 16 16 30 30 30 F F F 13 260-279 18 18 18 37 37 37 F F F 14 280-299 18 18 18 37 37 37 F F F 15 300-319 18 18 18 37 37 37 F F F 16 320-339 16 16 16 34 34 34 F F F 17 340-359 18 18 18 38 38 38 F F F 18 360-379 18 18 18 38 38 38 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 18 18 18 37 37 37 F F F 22 440-459 18 18 18 40 40 40 T T T 23 460-479 16 16 16 32 32 32 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 12 12 12 22 22 22 T T T 28 560-579 14 14 14 27 27 27 F F F 29 580-599 18 18 18 38 38 38 T T T Max gen: 18, Max flush: 40 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 F F F 1 020-039 19 19 19 41 41 41 F F F 2 040-059 18 18 18 40 40 40 F F F 3 060-079 18 18 18 39 39 39 F F F 4 080-099 16 16 16 30 30 30 F F F 5 100-119 18 18 18 39 39 39 F F F 6 120-139 16 16 16 33 33 33 F F F 7 140-159 16 16 16 33 33 33 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 12 12 12 22 22 22 F F F 10 200-219 18 18 18 38 38 38 F F F 11 220-239 18 18 18 38 38 38 F F F 12 240-259 16 16 16 30 30 30 F F F 13 260-279 18 18 18 37 37 37 F F F 14 280-299 18 18 18 37 37 37 F F F 15 300-319 18 18 18 37 37 37 F F F 16 320-339 16 16 16 34 34 34 F F F 17 340-359 18 18 18 38 38 38 F F F 18 360-379 18 18 18 38 38 38 F F F 19 380-399 8 8 8 13 13 13 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 18 18 18 37 37 37 F F F 22 440-459 19 19 19 41 41 41 F F F 23 460-479 16 16 16 32 32 32 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 19 19 19 41 41 41 F F F 28 560-579 14 14 14 27 27 27 F F F 29 580-599 19 19 19 41 41 41 F F F Max gen: 19, Max flush: 41 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.018006895Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.062802059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.063512503Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.063559042Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.063574002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.063589012Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272} {"msg":"Crucible 21ccb24c-f5b0-466d-b14d-601e3c3ebd2e has session id: 03ec5b34-2dd3-460a-9737-4b066420ab05","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064161137Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Upstairs opts: Upstairs UUID: 21ccb24c-f5b0-466d-b14d-601e3c3ebd2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064200547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Crucible stats registered with UUID: 21ccb24c-f5b0-466d-b14d-601e3c3ebd2e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064223687Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:01.064244686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064674583Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064701822Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272} {"msg":"21ccb24c-f5b0-466d-b14d-601e3c3ebd2e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064758912Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064777002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"0","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064806711Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"0","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064826151Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"1","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.064852421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"1","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06487763Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"2","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06490396Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"2","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.0649301Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06495476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06497854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"ds_connection connected from Ok(127.0.0.1:46659)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.065003829Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"ds_connection connected from Ok(127.0.0.1:53102)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.065162598Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"ds_connection connected from Ok(127.0.0.1:33655)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.065205368Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.065665094Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"0","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.065703853Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"1","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.065786802Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"2","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.066374357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"0","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.066414507Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"1","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.066449886Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"2","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067005461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067031651Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[0]R flush_numbers[0..12]: [27, 41, 40, 39, 30, 39, 33, 33, 30, 22, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067103201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[0]R generation[0..12]: [14, 19, 18, 18, 16, 18, 16, 16, 16, 12, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06712433Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06713919Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[1]R flush_numbers[0..12]: [27, 41, 40, 39, 30, 39, 33, 33, 30, 22, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06715293Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[1]R generation[0..12]: [14, 19, 18, 18, 16, 18, 16, 16, 16, 12, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.0671665Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06718056Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[2]R flush_numbers[0..12]: [27, 41, 40, 39, 30, 39, 33, 33, 30, 22, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.06719402Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[2]R generation[0..12]: [14, 19, 18, 18, 16, 18, 16, 16, 16, 12, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067207649Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067221619Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067234569Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:42:01.067247519Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067268289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067280909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"":"downstairs","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067294059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067305849Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"0","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067319279Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"1","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067332398Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"client":"2","":"downstairs","session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067345528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067357628Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"21ccb24c-f5b0-466d-b14d-601e3c3ebd2e is now active with session: 03ec5b34-2dd3-460a-9737-4b066420ab05","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067370018Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067381698Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272,"session_id":"03ec5b34-2dd3-460a-9737-4b066420ab05"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067393748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272} {"msg":"Activated sub_volume 21ccb24c-f5b0-466d-b14d-601e3c3ebd2e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:01.067404738Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 216 len 40960 data: 4 5 6 6 6 6 6 5 5 3 02/30 Write block 238 len 36864 data: 4 4 5 4 4 4 3 3 2 03/30 Read block 50 len 20480 04/30 Write block 337 len 16384 data: 4 4 4 4 05/30 Read block 309 len 16384 06/30 Read block 449 len 12288 07/30 Read block 241 len 16384 08/30 Read block 89 len 4096 09/30 Flush 10/30 Write block 579 len 40960 data: 5 5 4 3 3 4 4 3 4 3 11/30 Read block 241 len 28672 12/30 Write block 328 len 36864 data: 2 2 2 2 2 2 4 4 4 13/30 Read block 199 len 36864 14/30 Write block 461 len 24576 data: 3 3 3 3 3 3 15/30 Write block 73 len 36864 data: 5 5 4 4 4 3 3 3 3 16/30 Write block 385 len 40960 data: 2 2 2 2 2 2 2 2 2 2 17/30 Write block 282 len 12288 data: 5 4 4 18/30 Read block 168 len 4096 19/30 Write block 369 len 40960 data: 3 3 4 4 4 4 3 3 2 2 20/30 Flush 21/30 Write block 571 len 8192 data: 3 3 22/30 Read block 370 len 24576 23/30 Read block 15 len 32768 24/30 Read block 235 len 16384 25/30 Write block 181 len 8192 data: 2 2 26/30 Write block 356 len 20480 data: 3 4 3 3 3 27/30 Read block 362 len 4096 28/30 Write block 6 len 32768 data: 3 4 3 3 4 5 4 3 29/30 Read block 465 len 4096 30/30 Read block 103 len 40960 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 2 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 10 10 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:01.088616286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 27 27 27 T T T 1 020-039 19 19 19 41 41 41 F F F 2 040-059 18 18 18 40 40 40 F F F 3 060-079 20 20 20 43 43 43 F F F 4 080-099 20 20 20 43 43 43 F F F 5 100-119 18 18 18 39 39 39 F F F 6 120-139 16 16 16 33 33 33 F F F 7 140-159 16 16 16 33 33 33 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 12 12 12 22 22 22 T T T 10 200-219 20 20 20 42 42 42 F F F 11 220-239 20 20 20 42 42 42 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 18 18 18 37 37 37 F F F 14 280-299 20 20 20 43 43 43 F F F 15 300-319 18 18 18 37 37 37 F F F 16 320-339 20 20 20 43 43 43 F F F 17 340-359 20 20 20 42 42 42 T T T 18 360-379 20 20 20 43 43 43 T T T 19 380-399 20 20 20 43 43 43 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 18 18 18 37 37 37 F F F 22 440-459 19 19 19 41 41 41 F F F 23 460-479 20 20 20 43 43 43 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 19 19 19 41 41 41 F F F 28 560-579 20 20 20 43 43 43 T T T 29 580-599 20 20 20 43 43 43 F F F Max gen: 20, Max flush: 43 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 44 44 44 F F F 1 020-039 19 19 19 41 41 41 F F F 2 040-059 18 18 18 40 40 40 F F F 3 060-079 20 20 20 43 43 43 F F F 4 080-099 20 20 20 43 43 43 F F F 5 100-119 18 18 18 39 39 39 F F F 6 120-139 16 16 16 33 33 33 F F F 7 140-159 16 16 16 33 33 33 F F F 8 160-179 16 16 16 30 30 30 F F F 9 180-199 21 21 21 44 44 44 F F F 10 200-219 20 20 20 42 42 42 F F F 11 220-239 20 20 20 42 42 42 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 18 18 18 37 37 37 F F F 14 280-299 20 20 20 43 43 43 F F F 15 300-319 18 18 18 37 37 37 F F F 16 320-339 20 20 20 43 43 43 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 21 21 21 44 44 44 F F F 19 380-399 20 20 20 43 43 43 F F F 20 400-419 18 18 18 38 38 38 F F F 21 420-439 18 18 18 37 37 37 F F F 22 440-459 19 19 19 41 41 41 F F F 23 460-479 20 20 20 43 43 43 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 11 11 11 18 18 18 F F F 27 540-559 19 19 19 41 41 41 F F F 28 560-579 21 21 21 44 44 44 F F F 29 580-599 20 20 20 43 43 43 F F F Max gen: 21, Max flush: 44 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.322003312Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:05.364809175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.365163202Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.396642417Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.397211962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.397247571Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.397260871Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.397274961Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280} {"msg":"Crucible 67032fb2-3986-44f5-8dcd-75a6861b55a7 has session id: fc0ab4cd-14cc-4899-a39c-5c832004aec4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.397831266Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Upstairs opts: Upstairs UUID: 67032fb2-3986-44f5-8dcd-75a6861b55a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.397872016Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Crucible stats registered with UUID: 67032fb2-3986-44f5-8dcd-75a6861b55a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.397893476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:05.397907825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398298732Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398323212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280} {"msg":"67032fb2-3986-44f5-8dcd-75a6861b55a7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398385721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398417291Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"0","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.3984735Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"0","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.39850832Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"1","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.39854564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"1","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398578439Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"2","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398614539Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"2","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398647039Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398680289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398711548Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34946)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398750278Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"ds_connection connected from Ok(127.0.0.1:47809)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398782598Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"ds_connection connected from Ok(127.0.0.1:44658)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.398810577Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.399429962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"1","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.399460402Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"2","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.399486031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"0","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.400214445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"1","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.400323244Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"0","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.400358203Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"2","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.400933668Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.400959178Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[0]R flush_numbers[0..12]: [44, 41, 40, 43, 43, 39, 33, 33, 30, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401019097Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[0]R generation[0..12]: [21, 19, 18, 20, 20, 18, 16, 16, 16, 21, 20, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401038497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401052577Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[1]R flush_numbers[0..12]: [44, 41, 40, 43, 43, 39, 33, 33, 30, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401065927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[1]R generation[0..12]: [21, 19, 18, 20, 20, 18, 16, 16, 16, 21, 20, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401083447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401098207Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[2]R flush_numbers[0..12]: [44, 41, 40, 43, 43, 39, 33, 33, 30, 44, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401111987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[2]R generation[0..12]: [21, 19, 18, 20, 20, 18, 16, 16, 16, 21, 20, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401125356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401138856Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401151486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401164326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401177476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401199616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401227215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401239885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"0","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401253275Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"1","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401268885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"client":"2","":"downstairs","session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401292655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401308705Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"67032fb2-3986-44f5-8dcd-75a6861b55a7 is now active with session: fc0ab4cd-14cc-4899-a39c-5c832004aec4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401340445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401353704Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280,"session_id":"fc0ab4cd-14cc-4899-a39c-5c832004aec4"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401365754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280} {"msg":"Activated sub_volume 67032fb2-3986-44f5-8dcd-75a6861b55a7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:05.401382654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 89 len 28672 data: 4 4 4 4 3 3 2 02/30 Write block 54 len 12288 data: 6 6 5 03/30 Write block 322 len 24576 data: 3 3 3 3 2 2 04/30 Read block 202 len 36864 05/30 Read block 23 len 8192 06/30 Read block 371 len 16384 07/30 Write block 396 len 32768 data: 3 3 3 3 4 4 4 4 08/30 Flush 09/30 Flush 10/30 Write block 401 len 12288 data: 5 5 5 11/30 Flush 12/30 Write block 363 len 4096 data: 3 13/30 Write block 76 len 36864 data: 5 5 4 4 4 4 2 2 2 14/30 Read block 147 len 40960 15/30 Write block 176 len 8192 data: 4 3 16/30 Flush 17/30 Write block 555 len 12288 data: 3 2 3 18/30 Read block 178 len 40960 19/30 Flush 20/30 Write block 208 len 8192 data: 3 3 21/30 Write block 538 len 24576 data: 4 4 5 5 4 4 22/30 Write block 323 len 36864 data: 4 4 4 3 3 3 3 3 3 23/30 Flush 24/30 Write block 111 len 4096 data: 2 25/30 Read block 499 len 12288 26/30 Read block 112 len 20480 27/30 Read block 323 len 36864 28/30 Read block 209 len 12288 29/30 Write block 152 len 4096 data: 4 30/30 Write block 396 len 36864 data: 4 4 4 4 5 6 6 6 4 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:05.420245654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 44 44 44 F F F 1 020-039 19 19 19 41 41 41 F F F 2 040-059 22 22 22 45 45 45 F F F 3 060-079 22 22 22 48 48 48 F F F 4 080-099 22 22 22 48 48 48 F F F 5 100-119 18 18 18 39 39 39 T T T 6 120-139 16 16 16 33 33 33 F F F 7 140-159 16 16 16 33 33 33 T T T 8 160-179 22 22 22 48 48 48 F F F 9 180-199 21 21 21 44 44 44 F F F 10 200-219 22 22 22 50 50 50 F F F 11 220-239 20 20 20 42 42 42 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 18 18 18 37 37 37 F F F 14 280-299 20 20 20 43 43 43 F F F 15 300-319 18 18 18 37 37 37 F F F 16 320-339 22 22 22 50 50 50 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 22 22 22 48 48 48 F F F 19 380-399 22 22 22 45 45 45 T T T 20 400-419 22 22 22 47 47 47 T T T 21 420-439 18 18 18 37 37 37 F F F 22 440-459 19 19 19 41 41 41 F F F 23 460-479 20 20 20 43 43 43 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 22 22 22 50 50 50 F F F 27 540-559 22 22 22 50 50 50 F F F 28 560-579 21 21 21 44 44 44 F F F 29 580-599 20 20 20 43 43 43 F F F Max gen: 22, Max flush: 50 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 44 44 44 F F F 1 020-039 19 19 19 41 41 41 F F F 2 040-059 22 22 22 45 45 45 F F F 3 060-079 22 22 22 48 48 48 F F F 4 080-099 22 22 22 48 48 48 F F F 5 100-119 23 23 23 51 51 51 F F F 6 120-139 16 16 16 33 33 33 F F F 7 140-159 23 23 23 51 51 51 F F F 8 160-179 22 22 22 48 48 48 F F F 9 180-199 21 21 21 44 44 44 F F F 10 200-219 22 22 22 50 50 50 F F F 11 220-239 20 20 20 42 42 42 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 18 18 18 37 37 37 F F F 14 280-299 20 20 20 43 43 43 F F F 15 300-319 18 18 18 37 37 37 F F F 16 320-339 22 22 22 50 50 50 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 22 22 22 48 48 48 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 23 23 23 51 51 51 F F F 21 420-439 18 18 18 37 37 37 F F F 22 440-459 19 19 19 41 41 41 F F F 23 460-479 20 20 20 43 43 43 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 11 11 11 18 18 18 F F F 26 520-539 22 22 22 50 50 50 F F F 27 540-559 22 22 22 50 50 50 F F F 28 560-579 21 21 21 44 44 44 F F F 29 580-599 20 20 20 43 43 43 F F F Max gen: 23, Max flush: 51 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.687747701Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.732637225Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.733370058Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.733424348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.733457217Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.733475487Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288} {"msg":"Crucible 5f4297a2-a22f-4787-a4d9-2a740ccd7379 has session id: ddde97b7-0066-4aa2-a87e-03d93b36aa15","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734000292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Upstairs opts: Upstairs UUID: 5f4297a2-a22f-4787-a4d9-2a740ccd7379, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734040662Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Crucible stats registered with UUID: 5f4297a2-a22f-4787-a4d9-2a740ccd7379","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734064202Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:09.734084632Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734493808Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734520458Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288} {"msg":"5f4297a2-a22f-4787-a4d9-2a740ccd7379 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734577877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734595847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"0","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734629967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"0","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734654807Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"1","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734681597Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"1","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734707466Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"2","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734740036Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"2","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734767786Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734798205Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734825635Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"ds_connection connected from Ok(127.0.0.1:40254)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734851935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"ds_connection connected from Ok(127.0.0.1:63868)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734959084Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"ds_connection connected from Ok(127.0.0.1:65218)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.734999903Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.735574328Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"0","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.735602578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"2","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.735617848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"1","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736169073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"0","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736216272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"1","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736371161Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"2","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736824987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736859517Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[0]R flush_numbers[0..12]: [44, 41, 45, 48, 48, 51, 33, 51, 48, 44, 50, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736935576Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[0]R generation[0..12]: [21, 19, 22, 22, 22, 23, 16, 23, 22, 21, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736956056Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736971296Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[1]R flush_numbers[0..12]: [44, 41, 45, 48, 48, 51, 33, 51, 48, 44, 50, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736984896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[1]R generation[0..12]: [21, 19, 22, 22, 22, 23, 16, 23, 22, 21, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.736998676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737013235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[2]R flush_numbers[0..12]: [44, 41, 45, 48, 48, 51, 33, 51, 48, 44, 50, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737026535Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[2]R generation[0..12]: [21, 19, 22, 22, 22, 23, 16, 23, 22, 21, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737039995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737053425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737081605Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737095835Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737108664Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737121164Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"":"downstairs","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737133954Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737149474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"0","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:42:09.737162594Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"1","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737178494Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"client":"2","":"downstairs","session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737191344Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737203184Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"5f4297a2-a22f-4787-a4d9-2a740ccd7379 is now active with session: ddde97b7-0066-4aa2-a87e-03d93b36aa15","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737215694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737227333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288,"session_id":"ddde97b7-0066-4aa2-a87e-03d93b36aa15"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737240673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288} {"msg":"Activated sub_volume 5f4297a2-a22f-4787-a4d9-2a740ccd7379","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:09.737252213Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 202 len 8192 data: 2 3 03/30 Read block 43 len 4096 04/30 Read block 34 len 8192 05/30 Flush 06/30 Write block 61 len 36864 data: 4 4 4 4 4 3 3 3 3 07/30 Write block 431 len 40960 data: 4 3 3 3 3 3 3 2 2 2 08/30 Write block 28 len 20480 data: 3 2 2 2 2 09/30 Write block 507 len 32768 data: 3 3 3 3 3 3 3 2 10/30 Read block 500 len 20480 11/30 Read block 228 len 32768 12/30 Read block 274 len 8192 13/30 Write block 587 len 32768 data: 5 4 2 2 2 2 2 2 14/30 Write block 329 len 40960 data: 4 4 4 3 3 5 5 5 5 5 15/30 Read block 316 len 28672 16/30 Flush 17/30 Write block 3 len 36864 data: 2 3 3 4 5 4 4 5 6 18/30 Read block 460 len 4096 19/30 Read block 78 len 36864 20/30 Write block 229 len 12288 data: 6 5 5 21/30 Write block 32 len 36864 data: 3 2 2 2 3 4 3 4 6 22/30 Read block 21 len 20480 23/30 Read block 124 len 4096 24/30 Read block 231 len 40960 25/30 Flush 26/30 Write block 103 len 4096 data: 3 27/30 Write block 371 len 20480 data: 5 5 5 5 4 28/30 Write block 117 len 28672 data: 4 3 3 4 4 3 4 29/30 Write block 564 len 8192 data: 3 4 30/30 Write block 545 len 36864 data: 4 4 4 4 4 4 3 2 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:09.756867546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 55 55 55 F F F 1 020-039 24 24 24 55 55 55 F F F 2 040-059 24 24 24 55 55 55 F F F 3 060-079 24 24 24 54 54 54 F F F 4 080-099 22 22 22 48 48 48 F F F 5 100-119 23 23 23 51 51 51 T T T 6 120-139 16 16 16 33 33 33 T T T 7 140-159 23 23 23 51 51 51 F F F 8 160-179 22 22 22 48 48 48 F F F 9 180-199 21 21 21 44 44 44 F F F 10 200-219 24 24 24 53 53 53 F F F 11 220-239 24 24 24 55 55 55 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 18 18 18 37 37 37 F F F 14 280-299 20 20 20 43 43 43 F F F 15 300-319 18 18 18 37 37 37 F F F 16 320-339 24 24 24 54 54 54 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 22 22 22 48 48 48 T T T 19 380-399 23 23 23 51 51 51 F F F 20 400-419 23 23 23 51 51 51 F F F 21 420-439 24 24 24 54 54 54 F F F 22 440-459 24 24 24 54 54 54 F F F 23 460-479 20 20 20 43 43 43 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 24 24 24 54 54 54 F F F 26 520-539 22 22 22 50 50 50 F F F 27 540-559 22 22 22 50 50 50 T T T 28 560-579 21 21 21 44 44 44 T T T 29 580-599 24 24 24 54 54 54 F F F Max gen: 24, Max flush: 55 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 55 55 55 F F F 1 020-039 24 24 24 55 55 55 F F F 2 040-059 24 24 24 55 55 55 F F F 3 060-079 24 24 24 54 54 54 F F F 4 080-099 22 22 22 48 48 48 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 25 25 25 56 56 56 F F F 7 140-159 23 23 23 51 51 51 F F F 8 160-179 22 22 22 48 48 48 F F F 9 180-199 21 21 21 44 44 44 F F F 10 200-219 24 24 24 53 53 53 F F F 11 220-239 24 24 24 55 55 55 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 18 18 18 37 37 37 F F F 14 280-299 20 20 20 43 43 43 F F F 15 300-319 18 18 18 37 37 37 F F F 16 320-339 24 24 24 54 54 54 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 25 25 25 56 56 56 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 23 23 23 51 51 51 F F F 21 420-439 24 24 24 54 54 54 F F F 22 440-459 24 24 24 54 54 54 F F F 23 460-479 20 20 20 43 43 43 F F F 24 480-499 18 18 18 40 40 40 F F F 25 500-519 24 24 24 54 54 54 F F F 26 520-539 22 22 22 50 50 50 F F F 27 540-559 25 25 25 56 56 56 F F F 28 560-579 25 25 25 56 56 56 F F F 29 580-599 24 24 24 54 54 54 F F F Max gen: 25, Max flush: 56 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:13.998869304Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:14.042491449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.042846276Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.074632758Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.075245503Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.075281432Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.075295862Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.075310142Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296} {"msg":"Crucible e787cf77-e6e7-41d2-bd20-f09b4e18ef1f has session id: f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.075884577Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Upstairs opts: Upstairs UUID: e787cf77-e6e7-41d2-bd20-f09b4e18ef1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.075922377Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Crucible stats registered with UUID: e787cf77-e6e7-41d2-bd20-f09b4e18ef1f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.075942317Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:14.075955766Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076434932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076457982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296} {"msg":"e787cf77-e6e7-41d2-bd20-f09b4e18ef1f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076526701Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076544171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"0","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076572301Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"0","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076585841Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"1","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.0765997Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"1","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076612751Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"2","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.0766263Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"2","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.07664593Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.07666801Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.07669192Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55244)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076730869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:37316)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076760399Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51597)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.076784209Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.077582062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"2","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.077612971Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"0","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.077627881Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"1","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.078252656Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"0","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.078381055Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"1","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.078410374Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"2","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.078938599Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079049038Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[0]R flush_numbers[0..12]: [55, 55, 55, 54, 48, 56, 56, 51, 48, 44, 53, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079119758Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[0]R generation[0..12]: [24, 24, 24, 24, 22, 25, 25, 23, 22, 21, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079139788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079156057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[1]R flush_numbers[0..12]: [55, 55, 55, 54, 48, 56, 56, 51, 48, 44, 53, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079170087Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[1]R generation[0..12]: [24, 24, 24, 24, 22, 25, 25, 23, 22, 21, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079183787Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079197527Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[2]R flush_numbers[0..12]: [55, 55, 55, 54, 48, 56, 56, 51, 48, 44, 53, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079210917Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[2]R generation[0..12]: [24, 24, 24, 24, 22, 25, 25, 23, 22, 21, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079228797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079243227Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079255916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079268776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079281336Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079293846Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"":"downstairs","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:42:14.079306806Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079321946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"0","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079335176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"1","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079348016Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"client":"2","":"downstairs","session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079360946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079372946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"e787cf77-e6e7-41d2-bd20-f09b4e18ef1f is now active with session: f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079385425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079397025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296,"session_id":"f1ac1780-9ac1-4c45-9d9e-e1c7a60a26f6"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079408965Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296} {"msg":"Activated sub_volume e787cf77-e6e7-41d2-bd20-f09b4e18ef1f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:14.079419825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 40960 data: 4 4 3 3 3 3 3 2 2 2 02/30 Flush 03/30 Write block 489 len 24576 data: 2 2 3 4 4 4 04/30 Write block 43 len 40960 data: 5 4 3 2 3 4 4 4 4 6 05/30 Read block 304 len 28672 06/30 Write block 154 len 12288 data: 4 5 5 07/30 Write block 549 len 4096 data: 5 08/30 Flush 09/30 Write block 300 len 20480 data: 3 3 4 4 4 10/30 Read block 386 len 28672 11/30 Read block 336 len 16384 12/30 Flush 13/30 Flush 14/30 Write block 16 len 16384 data: 3 3 3 3 15/30 Read block 462 len 40960 16/30 Write block 264 len 32768 data: 2 2 2 2 2 2 2 2 17/30 Write block 465 len 28672 data: 4 4 3 3 3 3 3 18/30 Flush 19/30 Write block 412 len 4096 data: 2 20/30 Read block 439 len 4096 21/30 Read block 251 len 20480 22/30 Write block 500 len 40960 data: 3 3 2 2 2 2 2 4 4 4 23/30 Read block 352 len 16384 24/30 Write block 317 len 20480 data: 2 2 2 2 3 25/30 Write block 161 len 12288 data: 4 3 3 26/30 Write block 145 len 8192 data: 2 2 27/30 Write block 514 len 16384 data: 3 2 2 2 28/30 Read block 256 len 36864 29/30 Write block 525 len 24576 data: 3 3 4 3 3 3 30/30 Read block 449 len 16384 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:14.099453994Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 61 61 61 F F F 1 020-039 24 24 24 55 55 55 F F F 2 040-059 26 26 26 58 58 58 F F F 3 060-079 24 24 24 54 54 54 F F F 4 080-099 22 22 22 48 48 48 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 25 25 25 56 56 56 F F F 7 140-159 26 26 26 58 58 58 T T T 8 160-179 26 26 26 57 57 57 T T T 9 180-199 21 21 21 44 44 44 F F F 10 200-219 24 24 24 53 53 53 F F F 11 220-239 24 24 24 55 55 55 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 26 26 26 61 61 61 F F F 14 280-299 20 20 20 43 43 43 F F F 15 300-319 26 26 26 59 59 59 T T T 16 320-339 24 24 24 54 54 54 T T T 17 340-359 21 21 21 44 44 44 F F F 18 360-379 25 25 25 56 56 56 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 23 23 23 51 51 51 T T T 21 420-439 24 24 24 54 54 54 F F F 22 440-459 24 24 24 54 54 54 F F F 23 460-479 26 26 26 61 61 61 F F F 24 480-499 26 26 26 58 58 58 F F F 25 500-519 24 24 24 54 54 54 T T T 26 520-539 22 22 22 50 50 50 T T T 27 540-559 26 26 26 58 58 58 F F F 28 560-579 25 25 25 56 56 56 F F F 29 580-599 24 24 24 54 54 54 F F F Max gen: 26, Max flush: 61 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 61 61 61 F F F 1 020-039 24 24 24 55 55 55 F F F 2 040-059 26 26 26 58 58 58 F F F 3 060-079 24 24 24 54 54 54 F F F 4 080-099 22 22 22 48 48 48 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 25 25 25 56 56 56 F F F 7 140-159 27 27 27 62 62 62 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 21 21 21 44 44 44 F F F 10 200-219 24 24 24 53 53 53 F F F 11 220-239 24 24 24 55 55 55 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 26 26 26 61 61 61 F F F 14 280-299 20 20 20 43 43 43 F F F 15 300-319 27 27 27 62 62 62 F F F 16 320-339 27 27 27 62 62 62 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 25 25 25 56 56 56 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 27 27 27 62 62 62 F F F 21 420-439 24 24 24 54 54 54 F F F 22 440-459 24 24 24 54 54 54 F F F 23 460-479 26 26 26 61 61 61 F F F 24 480-499 26 26 26 58 58 58 F F F 25 500-519 27 27 27 62 62 62 F F F 26 520-539 27 27 27 62 62 62 F F F 27 540-559 26 26 26 58 58 58 F F F 28 560-579 25 25 25 56 56 56 F F F 29 580-599 24 24 24 54 54 54 F F F Max gen: 27, Max flush: 62 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.371793598Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.417804702Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.418388726Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.418433226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.418447776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.418466316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304} {"msg":"Crucible fd39c30b-99fe-4c50-9bc3-8928b04dd81a has session id: 4f545b97-c00c-4f50-9751-ff73050cd765","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419179299Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Upstairs opts: Upstairs UUID: fd39c30b-99fe-4c50-9bc3-8928b04dd81a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419208639Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Crucible stats registered with UUID: fd39c30b-99fe-4c50-9bc3-8928b04dd81a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419223079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:18.419235579Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419770784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419802534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304} {"msg":"fd39c30b-99fe-4c50-9bc3-8928b04dd81a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419887683Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419915742Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"0","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419961682Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"0","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.419989552Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"1","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420016382Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"1","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420043281Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"2","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420066681Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"2","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420080991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420099051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420113461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"ds_connection connected from Ok(127.0.0.1:40463)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420127211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"ds_connection connected from Ok(127.0.0.1:63112)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420307959Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"ds_connection connected from Ok(127.0.0.1:52638)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420353869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420814264Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"2","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420904373Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"0","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.420980933Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"1","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.421486658Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"2","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.421525978Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"0","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.421736196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"1","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422107333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422201882Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[0]R flush_numbers[0..12]: [61, 55, 58, 54, 48, 56, 56, 62, 62, 44, 53, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422326181Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[0]R generation[0..12]: [26, 24, 26, 24, 22, 25, 25, 27, 27, 21, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.42234916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.4223644Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[1]R flush_numbers[0..12]: [61, 55, 58, 54, 48, 56, 56, 62, 62, 44, 53, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.42237814Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[1]R generation[0..12]: [26, 24, 26, 24, 22, 25, 25, 27, 27, 21, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.42239183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.42240568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[2]R flush_numbers[0..12]: [61, 55, 58, 54, 48, 56, 56, 62, 62, 44, 53, 55]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.42241898Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[2]R generation[0..12]: [26, 24, 26, 24, 22, 25, 25, 27, 27, 21, 24, 24]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.42243206Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.42244561Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422458239Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422471269Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422483799Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422496639Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"":"downstairs","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422509409Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422521079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"0","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422534059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"1","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"Wait for a query_work_queue command to finish before sending IO msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422557339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"client":"2","":"downstairs","session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422570918Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422582798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"fd39c30b-99fe-4c50-9bc3-8928b04dd81a is now active with session: 4f545b97-c00c-4f50-9751-ff73050cd765","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422595288Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422606788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304,"session_id":"4f545b97-c00c-4f50-9751-ff73050cd765"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422618608Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304} {"msg":"Activated sub_volume fd39c30b-99fe-4c50-9bc3-8928b04dd81a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:18.422629448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 235 len 4096 data: 6 03/30 Flush 04/30 Flush 05/30 Write block 581 len 16384 data: 5 4 4 5 06/30 Read block 76 len 32768 07/30 Read block 69 len 8192 08/30 Write block 575 len 40960 data: 2 3 3 5 6 6 6 5 5 6 09/30 Write block 140 len 24576 data: 3 3 3 2 2 3 10/30 Read block 219 len 28672 11/30 Write block 196 len 24576 data: 3 3 3 3 3 2 12/30 Write block 281 len 4096 data: 5 13/30 Read block 179 len 4096 14/30 Read block 505 len 32768 15/30 Write block 121 len 28672 data: 5 4 5 4 2 2 2 16/30 Read block 494 len 20480 17/30 Write block 35 len 4096 data: 3 18/30 Write block 7 len 12288 data: 6 5 5 19/30 Read block 579 len 40960 20/30 Read block 180 len 4096 21/30 Read block 547 len 32768 22/30 Read block 259 len 24576 23/30 Write block 470 len 36864 data: 4 4 3 3 3 3 3 4 4 24/30 Write block 76 len 12288 data: 6 6 5 25/30 Read block 399 len 32768 26/30 Read block 545 len 4096 27/30 Write block 411 len 28672 data: 2 3 2 2 2 4 4 28/30 Write block 360 len 24576 data: 4 3 3 4 3 2 29/30 Write block 290 len 28672 data: 3 3 2 2 3 3 3 30/30 Read block 33 len 36864 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 4 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 25 25 26 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:18.441916194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 61 61 61 T T T 1 020-039 24 24 24 55 55 55 T T T 2 040-059 26 26 26 58 58 58 F F F 3 060-079 24 24 24 54 54 54 T T T 4 080-099 22 22 22 48 48 48 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 25 25 25 56 56 56 T T T 7 140-159 27 27 27 62 62 62 T T T 8 160-179 27 27 27 62 62 62 F F F 9 180-199 21 21 21 44 44 44 T T T 10 200-219 24 24 24 53 53 53 T T T 11 220-239 28 28 28 64 64 64 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 26 26 26 61 61 61 F F F 14 280-299 20 20 20 43 43 43 T T T 15 300-319 27 27 27 62 62 62 F F F 16 320-339 27 27 27 62 62 62 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 25 25 25 56 56 56 T T T 19 380-399 23 23 23 51 51 51 F F F 20 400-419 27 27 27 62 62 62 T T T 21 420-439 24 24 24 54 54 54 F F F 22 440-459 24 24 24 54 54 54 F F F 23 460-479 26 26 26 61 61 61 T T T 24 480-499 26 26 26 58 58 58 F F F 25 500-519 27 27 27 62 62 62 F F F 26 520-539 27 27 27 62 62 62 F F F 27 540-559 26 26 26 58 58 58 F F F 28 560-579 25 25 25 56 56 56 T T T 29 580-599 24 24 24 54 54 54 T T T Max gen: 28, Max flush: 64 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 29 29 29 65 65 65 F F F 1 020-039 29 29 29 65 65 65 F F F 2 040-059 26 26 26 58 58 58 F F F 3 060-079 29 29 29 65 65 65 F F F 4 080-099 22 22 22 48 48 48 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 29 29 29 65 65 65 F F F 7 140-159 29 29 29 65 65 65 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 29 29 29 65 65 65 F F F 10 200-219 29 29 29 65 65 65 F F F 11 220-239 28 28 28 64 64 64 F F F 12 240-259 20 20 20 42 42 42 F F F 13 260-279 26 26 26 61 61 61 F F F 14 280-299 29 29 29 65 65 65 F F F 15 300-319 27 27 27 62 62 62 F F F 16 320-339 27 27 27 62 62 62 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 29 29 29 65 65 65 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 29 29 29 65 65 65 F F F 21 420-439 24 24 24 54 54 54 F F F 22 440-459 24 24 24 54 54 54 F F F 23 460-479 29 29 29 65 65 65 F F F 24 480-499 26 26 26 58 58 58 F F F 25 500-519 27 27 27 62 62 62 F F F 26 520-539 27 27 27 62 62 62 F F F 27 540-559 26 26 26 58 58 58 F F F 28 560-579 29 29 29 65 65 65 F F F 29 580-599 29 29 29 65 65 65 F F F Max gen: 29, Max flush: 65 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.718786776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:22.761798497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.762174684Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79349893Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.794113364Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.794147964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.794163514Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.794180584Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312} {"msg":"Crucible 58830609-f39b-4553-b1e3-bcc63d96667d has session id: 97bf3f1d-c45b-4e53-8329-2ce98598e9cd","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.794825988Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Upstairs opts: Upstairs UUID: 58830609-f39b-4553-b1e3-bcc63d96667d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.794861417Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Crucible stats registered with UUID: 58830609-f39b-4553-b1e3-bcc63d96667d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.794879077Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:22.794898307Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795243584Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795265304Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312} {"msg":"58830609-f39b-4553-b1e3-bcc63d96667d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795319013Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795345393Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"0","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795382333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"0","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795408463Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"1","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795436612Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"1","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795463552Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"2","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795492002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"2","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795520131Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795547311Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795575381Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:48055)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.795602511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:37357)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79562931Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:51540)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79565616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.796297844Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"1","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.796331064Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"2","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.796346754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"0","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797011828Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"1","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797053118Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"2","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797081497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"0","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797612853Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797639183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[0]R flush_numbers[0..12]: [65, 65, 58, 65, 48, 56, 65, 65, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797666102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[0]R generation[0..12]: [29, 29, 26, 29, 22, 25, 29, 29, 27, 29, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797681092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797695502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[1]R flush_numbers[0..12]: [65, 65, 58, 65, 48, 56, 65, 65, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797716102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[1]R generation[0..12]: [29, 29, 26, 29, 22, 25, 29, 29, 27, 29, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797735542Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797753261Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[2]R flush_numbers[0..12]: [65, 65, 58, 65, 48, 56, 65, 65, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797767161Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[2]R generation[0..12]: [29, 29, 26, 29, 22, 25, 29, 29, 27, 29, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797780511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797794081Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797806751Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797819541Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797832071Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79784475Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"":"downstairs","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79785767Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79786932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"0","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:42:22.79788217Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"1","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79789766Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"client":"2","":"downstairs","session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79791051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79792232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"58830609-f39b-4553-b1e3-bcc63d96667d is now active with session: 97bf3f1d-c45b-4e53-8329-2ce98598e9cd","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79793474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79794812Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312,"session_id":"97bf3f1d-c45b-4e53-8329-2ce98598e9cd"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.797960049Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312} {"msg":"Activated sub_volume 58830609-f39b-4553-b1e3-bcc63d96667d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:22.79797106Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 460 len 36864 data: 3 4 4 4 4 5 5 4 4 02/30 Write block 130 len 16384 data: 2 2 2 2 03/30 Read block 443 len 20480 04/30 Flush 05/30 Write block 122 len 40960 data: 5 6 5 3 3 3 2 2 3 3 06/30 Write block 81 len 24576 data: 5 3 3 3 2 3 07/30 Read block 22 len 4096 08/30 Read block 474 len 4096 09/30 Read block 108 len 28672 10/30 Read block 70 len 24576 11/30 Read block 190 len 28672 12/30 Write block 30 len 40960 data: 3 3 4 3 3 4 4 5 4 5 13/30 Write block 298 len 24576 data: 3 3 4 4 5 5 14/30 Read block 199 len 28672 15/30 Write block 10 len 20480 data: 6 7 5 4 3 16/30 Read block 469 len 8192 17/30 Read block 111 len 4096 18/30 Read block 172 len 32768 19/30 Write block 454 len 20480 data: 4 4 4 4 4 20/30 Read block 397 len 20480 21/30 Read block 227 len 28672 22/30 Write block 533 len 28672 data: 3 4 5 4 4 5 5 23/30 Write block 240 len 40960 data: 6 5 5 5 4 4 3 2 2 2 24/30 Write block 240 len 8192 data: 7 6 25/30 Flush 26/30 Write block 572 len 36864 data: 4 2 3 3 4 4 6 7 7 27/30 Write block 29 len 36864 data: 3 4 4 5 4 4 5 5 6 28/30 Write block 318 len 16384 data: 3 3 3 4 29/30 Read block 106 len 24576 30/30 Write block 413 len 36864 data: 3 3 3 5 5 5 5 5 4 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:22.819022499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 67 67 67 F F F 1 020-039 30 30 30 67 67 67 T T T 2 040-059 26 26 26 58 58 58 F F F 3 060-079 29 29 29 65 65 65 F F F 4 080-099 30 30 30 67 67 67 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 30 30 30 67 67 67 F F F 7 140-159 29 29 29 65 65 65 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 29 29 29 65 65 65 F F F 10 200-219 29 29 29 65 65 65 F F F 11 220-239 28 28 28 64 64 64 F F F 12 240-259 30 30 30 67 67 67 F F F 13 260-279 26 26 26 61 61 61 F F F 14 280-299 30 30 30 67 67 67 F F F 15 300-319 30 30 30 67 67 67 T T T 16 320-339 27 27 27 62 62 62 T T T 17 340-359 21 21 21 44 44 44 F F F 18 360-379 29 29 29 65 65 65 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 29 29 29 65 65 65 T T T 21 420-439 24 24 24 54 54 54 T T T 22 440-459 30 30 30 67 67 67 F F F 23 460-479 30 30 30 66 66 66 F F F 24 480-499 26 26 26 58 58 58 F F F 25 500-519 27 27 27 62 62 62 F F F 26 520-539 30 30 30 67 67 67 F F F 27 540-559 26 26 26 58 58 58 F F F 28 560-579 29 29 29 65 65 65 T T T 29 580-599 29 29 29 65 65 65 T T T Max gen: 30, Max flush: 67 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 67 67 67 F F F 1 020-039 31 31 31 68 68 68 F F F 2 040-059 26 26 26 58 58 58 F F F 3 060-079 29 29 29 65 65 65 F F F 4 080-099 30 30 30 67 67 67 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 30 30 30 67 67 67 F F F 7 140-159 29 29 29 65 65 65 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 29 29 29 65 65 65 F F F 10 200-219 29 29 29 65 65 65 F F F 11 220-239 28 28 28 64 64 64 F F F 12 240-259 30 30 30 67 67 67 F F F 13 260-279 26 26 26 61 61 61 F F F 14 280-299 30 30 30 67 67 67 F F F 15 300-319 31 31 31 68 68 68 F F F 16 320-339 31 31 31 68 68 68 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 29 29 29 65 65 65 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 31 31 31 68 68 68 F F F 22 440-459 30 30 30 67 67 67 F F F 23 460-479 30 30 30 66 66 66 F F F 24 480-499 26 26 26 58 58 58 F F F 25 500-519 27 27 27 62 62 62 F F F 26 520-539 30 30 30 67 67 67 F F F 27 540-559 26 26 26 58 58 58 F F F 28 560-579 31 31 31 68 68 68 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 31, Max flush: 68 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.097847635Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:27.141016714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.141381101Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.172712997Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.173301592Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.173336241Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.173353931Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.173368301Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320} {"msg":"Crucible 696e7345-a627-4718-bcc5-9dbd020d77ce has session id: af7dcc43-32e8-448e-a29c-5e1c9df89f35","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174002675Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Upstairs opts: Upstairs UUID: 696e7345-a627-4718-bcc5-9dbd020d77ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174042165Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Crucible stats registered with UUID: 696e7345-a627-4718-bcc5-9dbd020d77ce","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174066945Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:27.174087615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174473421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174500271Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320} {"msg":"696e7345-a627-4718-bcc5-9dbd020d77ce active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.17455137Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.17457804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"0","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.17460837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"0","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.17462344Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"1","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.17464721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"1","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.17467426Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"2","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174699029Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"2","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174722869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174747229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174771148Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"ds_connection connected from Ok(127.0.0.1:42000)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174795738Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"ds_connection connected from Ok(127.0.0.1:64698)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174821378Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"ds_connection connected from Ok(127.0.0.1:39439)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.174936257Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.175526182Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"1","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.175554561Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"2","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.17570659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"0","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.176298635Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"1","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.176332994Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"0","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.176352234Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"2","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.176929639Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.176970259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[0]R flush_numbers[0..12]: [67, 68, 58, 65, 67, 56, 67, 65, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177108198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[0]R generation[0..12]: [30, 31, 26, 29, 30, 25, 30, 29, 27, 29, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177127887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177142297Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[1]R flush_numbers[0..12]: [67, 68, 58, 65, 67, 56, 67, 65, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177156177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[1]R generation[0..12]: [30, 31, 26, 29, 30, 25, 30, 29, 27, 29, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177169657Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177183537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[2]R flush_numbers[0..12]: [67, 68, 58, 65, 67, 56, 67, 65, 62, 65, 65, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177196887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[2]R generation[0..12]: [30, 31, 26, 29, 30, 25, 30, 29, 27, 29, 29, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177210006Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177223706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177236186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177248916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177261306Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177273786Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"":"downstairs","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177297156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177308876Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"0","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177321605Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"1","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177334325Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"client":"2","":"downstairs","session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177347145Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177358965Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"696e7345-a627-4718-bcc5-9dbd020d77ce is now active with session: af7dcc43-32e8-448e-a29c-5e1c9df89f35","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177371425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177383005Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320,"session_id":"af7dcc43-32e8-448e-a29c-5e1c9df89f35"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177395035Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320} {"msg":"Activated sub_volume 696e7345-a627-4718-bcc5-9dbd020d77ce","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:27.177405935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 8192 data: 3 3 02/30 Read block 413 len 8192 03/30 Read block 67 len 4096 04/30 Read block 557 len 24576 05/30 Read block 310 len 24576 06/30 Flush 07/30 Flush 08/30 Write block 513 len 20480 data: 4 4 3 3 3 09/30 Read block 145 len 12288 10/30 Write block 15 len 32768 data: 4 4 4 4 4 3 2 2 11/30 Read block 210 len 36864 12/30 Read block 49 len 36864 13/30 Write block 327 len 20480 data: 4 4 5 5 5 14/30 Read block 127 len 20480 15/30 Write block 300 len 12288 data: 5 5 6 16/30 Read block 536 len 20480 17/30 Read block 93 len 16384 18/30 Read block 393 len 4096 19/30 Read block 454 len 8192 20/30 Write block 198 len 20480 data: 4 4 4 3 3 21/30 Read block 0 len 4096 22/30 Read block 423 len 32768 23/30 Read block 225 len 8192 24/30 Read block 154 len 20480 25/30 Write block 538 len 20480 data: 6 6 6 6 5 26/30 Read block 117 len 8192 27/30 Write block 308 len 12288 data: 2 2 2 28/30 Write block 463 len 40960 data: 5 5 6 6 5 5 4 5 5 4 29/30 Read block 53 len 40960 30/30 Write block 483 len 36864 data: 3 4 4 4 3 2 3 3 4 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:27.197579952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 67 67 67 T T T 1 020-039 31 31 31 68 68 68 T T T 2 040-059 26 26 26 58 58 58 F F F 3 060-079 29 29 29 65 65 65 F F F 4 080-099 30 30 30 67 67 67 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 30 30 30 67 67 67 F F F 7 140-159 29 29 29 65 65 65 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 29 29 29 65 65 65 T T T 10 200-219 29 29 29 65 65 65 T T T 11 220-239 28 28 28 64 64 64 F F F 12 240-259 30 30 30 67 67 67 F F F 13 260-279 32 32 32 69 69 69 F F F 14 280-299 30 30 30 67 67 67 F F F 15 300-319 31 31 31 68 68 68 T T T 16 320-339 31 31 31 68 68 68 T T T 17 340-359 21 21 21 44 44 44 F F F 18 360-379 29 29 29 65 65 65 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 31 31 31 68 68 68 F F F 22 440-459 30 30 30 67 67 67 F F F 23 460-479 30 30 30 66 66 66 T T T 24 480-499 26 26 26 58 58 58 T T T 25 500-519 27 27 27 62 62 62 T T T 26 520-539 30 30 30 67 67 67 T T T 27 540-559 26 26 26 58 58 58 T T T 28 560-579 31 31 31 68 68 68 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 32, Max flush: 69 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 33 33 33 70 70 70 F F F 1 020-039 33 33 33 70 70 70 F F F 2 040-059 26 26 26 58 58 58 F F F 3 060-079 29 29 29 65 65 65 F F F 4 080-099 30 30 30 67 67 67 F F F 5 100-119 25 25 25 56 56 56 F F F 6 120-139 30 30 30 67 67 67 F F F 7 140-159 29 29 29 65 65 65 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 33 33 33 70 70 70 F F F 11 220-239 28 28 28 64 64 64 F F F 12 240-259 30 30 30 67 67 67 F F F 13 260-279 32 32 32 69 69 69 F F F 14 280-299 30 30 30 67 67 67 F F F 15 300-319 33 33 33 70 70 70 F F F 16 320-339 33 33 33 70 70 70 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 29 29 29 65 65 65 F F F 19 380-399 23 23 23 51 51 51 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 31 31 31 68 68 68 F F F 22 440-459 30 30 30 67 67 67 F F F 23 460-479 33 33 33 70 70 70 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 33 33 33 70 70 70 F F F 26 520-539 33 33 33 70 70 70 F F F 27 540-559 33 33 33 70 70 70 F F F 28 560-579 31 31 31 68 68 68 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 33, Max flush: 70 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.497991782Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.54240591Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.543087974Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.543142263Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.543160803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.543179313Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328} {"msg":"Crucible 057ed965-fcdb-4476-a97f-0733cd94d600 has session id: e5f1d203-6c19-4c55-ac70-dc212edc8bff","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.543737608Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Upstairs opts: Upstairs UUID: 057ed965-fcdb-4476-a97f-0733cd94d600, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.543785858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Crucible stats registered with UUID: 057ed965-fcdb-4476-a97f-0733cd94d600","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.543815697Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:31.543847177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544405592Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544439862Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328} {"msg":"057ed965-fcdb-4476-a97f-0733cd94d600 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544531051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544566891Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"0","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.54461754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"0","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.5446447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"1","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.54467056Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"1","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544695Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"2","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544720089Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"2","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544745539Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544770149Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544793919Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"ds_connection connected from Ok(127.0.0.1:32831)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544819928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"ds_connection connected from Ok(127.0.0.1:55003)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544846798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"ds_connection connected from Ok(127.0.0.1:47864)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.544878008Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.545617111Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"2","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.545645251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"0","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.545660481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"1","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.546262885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"1","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.546291345Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"2","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.546306115Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"0","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.54688481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.546991889Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[0]R flush_numbers[0..12]: [70, 70, 58, 65, 67, 56, 67, 65, 62, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547059628Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[0]R generation[0..12]: [33, 33, 26, 29, 30, 25, 30, 29, 27, 33, 33, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547088218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547104748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[1]R flush_numbers[0..12]: [70, 70, 58, 65, 67, 56, 67, 65, 62, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547118998Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[1]R generation[0..12]: [33, 33, 26, 29, 30, 25, 30, 29, 27, 33, 33, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547132547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547149597Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[2]R flush_numbers[0..12]: [70, 70, 58, 65, 67, 56, 67, 65, 62, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547163317Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[2]R generation[0..12]: [33, 33, 26, 29, 30, 25, 30, 29, 27, 33, 33, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547176367Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547189897Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547202237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547215127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547235277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547264326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"":"downstairs","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547278776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547290426Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"0","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547303506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"1","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547325646Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"client":"2","":"downstairs","session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547346296Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547358845Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"057ed965-fcdb-4476-a97f-0733cd94d600 is now active with session: e5f1d203-6c19-4c55-ac70-dc212edc8bff","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547371435Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547385995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328,"session_id":"e5f1d203-6c19-4c55-ac70-dc212edc8bff"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547404245Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328} {"msg":"Activated sub_volume 057ed965-fcdb-4476-a97f-0733cd94d600","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:31.547424825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 278 len 20480 data: 4 6 6 6 6 02/30 Write block 375 len 16384 data: 5 4 3 3 03/30 Write block 255 len 16384 data: 4 4 5 5 04/30 Flush 05/30 Write block 386 len 20480 data: 3 3 3 3 3 06/30 Read block 283 len 24576 07/30 Read block 112 len 4096 08/30 Read block 539 len 32768 09/30 Read block 295 len 4096 10/30 Read block 198 len 16384 11/30 Read block 129 len 36864 12/30 Read block 36 len 12288 13/30 Write block 551 len 4096 data: 4 14/30 Write block 62 len 12288 data: 5 5 5 15/30 Write block 151 len 24576 data: 3 5 4 5 6 6 16/30 Read block 160 len 28672 17/30 Read block 157 len 24576 18/30 Write block 438 len 20480 data: 3 3 3 2 3 19/30 Write block 304 len 12288 data: 5 3 3 20/30 Read block 92 len 40960 21/30 Read block 80 len 28672 22/30 Read block 3 len 20480 23/30 Write block 19 len 16384 data: 5 4 3 3 24/30 Read block 24 len 28672 25/30 Read block 26 len 4096 26/30 Flush 27/30 Write block 435 len 16384 data: 4 4 4 4 28/30 Read block 90 len 20480 29/30 Write block 108 len 16384 data: 2 2 2 3 30/30 Write block 519 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:31.566373343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 34 34 34 72 72 72 F F F 2 040-059 26 26 26 58 58 58 F F F 3 060-079 34 34 34 72 72 72 F F F 4 080-099 30 30 30 67 67 67 F F F 5 100-119 25 25 25 56 56 56 T T T 6 120-139 30 30 30 67 67 67 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 33 33 33 70 70 70 F F F 11 220-239 28 28 28 64 64 64 F F F 12 240-259 34 34 34 71 71 71 F F F 13 260-279 34 34 34 71 71 71 F F F 14 280-299 34 34 34 71 71 71 F F F 15 300-319 34 34 34 72 72 72 F F F 16 320-339 33 33 33 70 70 70 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 34 34 34 71 71 71 F F F 19 380-399 34 34 34 72 72 72 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 34 34 34 72 72 72 T T T 22 440-459 34 34 34 72 72 72 F F F 23 460-479 33 33 33 70 70 70 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 33 33 33 70 70 70 T T T 26 520-539 33 33 33 70 70 70 F F F 27 540-559 34 34 34 72 72 72 F F F 28 560-579 31 31 31 68 68 68 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 34, Max flush: 72 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 34 34 34 72 72 72 F F F 2 040-059 26 26 26 58 58 58 F F F 3 060-079 34 34 34 72 72 72 F F F 4 080-099 30 30 30 67 67 67 F F F 5 100-119 35 35 35 73 73 73 F F F 6 120-139 30 30 30 67 67 67 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 33 33 33 70 70 70 F F F 11 220-239 28 28 28 64 64 64 F F F 12 240-259 34 34 34 71 71 71 F F F 13 260-279 34 34 34 71 71 71 F F F 14 280-299 34 34 34 71 71 71 F F F 15 300-319 34 34 34 72 72 72 F F F 16 320-339 33 33 33 70 70 70 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 34 34 34 71 71 71 F F F 19 380-399 34 34 34 72 72 72 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 35 35 35 73 73 73 F F F 22 440-459 34 34 34 72 72 72 F F F 23 460-479 33 33 33 70 70 70 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 35 35 35 73 73 73 F F F 26 520-539 33 33 33 70 70 70 F F F 27 540-559 34 34 34 72 72 72 F F F 28 560-579 31 31 31 68 68 68 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 35, Max flush: 73 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.800650272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.845402727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.84607896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.84614038Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.84616766Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.846193139Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336} {"msg":"Crucible 697fa51d-e726-4f9a-91f6-3630e419eea9 has session id: 2260e07b-cfd4-4244-a857-cce916679514","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.846714265Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Upstairs opts: Upstairs UUID: 697fa51d-e726-4f9a-91f6-3630e419eea9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.846754534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Crucible stats registered with UUID: 697fa51d-e726-4f9a-91f6-3630e419eea9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.846777804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:35.846798274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.8472196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.84724672Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336} {"msg":"697fa51d-e726-4f9a-91f6-3630e419eea9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.84727551Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847290039Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"0","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847319269Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"0","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847333069Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"1","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847347009Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"1","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847360289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"2","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847377199Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"2","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847430208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847480108Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847508217Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"ds_connection connected from Ok(127.0.0.1:45856)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847532587Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"ds_connection connected from Ok(127.0.0.1:42230)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847711206Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"ds_connection connected from Ok(127.0.0.1:64885)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.847753825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.84835755Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"0","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.848385489Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"1","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.848403889Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"2","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.849050874Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"1","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.849195152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"0","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.849281472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"2","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.849570349Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.849833247Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[0]R flush_numbers[0..12]: [72, 72, 58, 72, 67, 73, 67, 72, 62, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.849978445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[0]R generation[0..12]: [34, 34, 26, 34, 30, 35, 30, 34, 27, 33, 33, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850007685Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850026945Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[1]R flush_numbers[0..12]: [72, 72, 58, 72, 67, 73, 67, 72, 62, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850041434Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[1]R generation[0..12]: [34, 34, 26, 34, 30, 35, 30, 34, 27, 33, 33, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850054954Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850068544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[2]R flush_numbers[0..12]: [72, 72, 58, 72, 67, 73, 67, 72, 62, 70, 70, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850082174Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[2]R generation[0..12]: [34, 34, 26, 34, 30, 35, 30, 34, 27, 33, 33, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850095474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850108984Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850121404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850134344Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:42:35.850146894Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850163113Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"":"downstairs","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850176283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850187983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"0","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850201113Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"1","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850215903Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"client":"2","":"downstairs","session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850229273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850241193Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"697fa51d-e726-4f9a-91f6-3630e419eea9 is now active with session: 2260e07b-cfd4-4244-a857-cce916679514","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850253533Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850265092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336,"session_id":"2260e07b-cfd4-4244-a857-cce916679514"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850276903Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336} {"msg":"Activated sub_volume 697fa51d-e726-4f9a-91f6-3630e419eea9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:35.850287752Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 261 len 16384 data: 2 2 2 4 02/30 Write block 87 len 36864 data: 3 3 5 5 5 5 4 4 3 03/30 Write block 383 len 12288 data: 3 2 3 04/30 Read block 411 len 8192 05/30 Write block 44 len 36864 data: 5 4 3 4 5 5 5 5 7 06/30 Read block 258 len 36864 07/30 Read block 269 len 28672 08/30 Write block 283 len 8192 data: 5 5 09/30 Flush 10/30 Flush 11/30 Write block 366 len 4096 data: 3 12/30 Read block 111 len 28672 13/30 Write block 374 len 28672 data: 6 6 5 4 4 3 3 14/30 Read block 445 len 20480 15/30 Read block 460 len 28672 16/30 Read block 12 len 20480 17/30 Write block 316 len 24576 data: 3 3 4 4 4 5 18/30 Read block 296 len 24576 19/30 Read block 566 len 36864 20/30 Read block 382 len 8192 21/30 Write block 560 len 16384 data: 4 4 3 3 22/30 Read block 528 len 40960 23/30 Read block 584 len 4096 24/30 Flush 25/30 Write block 212 len 12288 data: 3 3 4 26/30 Read block 397 len 8192 27/30 Read block 354 len 28672 28/30 Write block 546 len 16384 data: 5 5 5 6 29/30 Write block 56 len 28672 data: 6 4 2 2 4 5 6 30/30 Write block 526 len 16384 data: 4 5 4 4 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:35.869784826Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 34 34 34 72 72 72 F F F 2 040-059 36 36 36 74 74 74 T T T 3 060-079 34 34 34 72 72 72 T T T 4 080-099 36 36 36 74 74 74 F F F 5 100-119 35 35 35 73 73 73 F F F 6 120-139 30 30 30 67 67 67 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 33 33 33 70 70 70 T T T 11 220-239 28 28 28 64 64 64 F F F 12 240-259 34 34 34 71 71 71 F F F 13 260-279 36 36 36 74 74 74 F F F 14 280-299 36 36 36 74 74 74 F F F 15 300-319 36 36 36 76 76 76 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 36 36 36 76 76 76 F F F 19 380-399 36 36 36 76 76 76 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 35 35 35 73 73 73 F F F 22 440-459 34 34 34 72 72 72 F F F 23 460-479 33 33 33 70 70 70 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 35 35 35 73 73 73 F F F 26 520-539 33 33 33 70 70 70 T T T 27 540-559 34 34 34 72 72 72 T T T 28 560-579 36 36 36 76 76 76 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 36, Max flush: 76 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 34 34 34 72 72 72 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 37 37 37 77 77 77 F F F 4 080-099 36 36 36 74 74 74 F F F 5 100-119 35 35 35 73 73 73 F F F 6 120-139 30 30 30 67 67 67 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 37 37 37 77 77 77 F F F 11 220-239 28 28 28 64 64 64 F F F 12 240-259 34 34 34 71 71 71 F F F 13 260-279 36 36 36 74 74 74 F F F 14 280-299 36 36 36 74 74 74 F F F 15 300-319 36 36 36 76 76 76 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 36 36 36 76 76 76 F F F 19 380-399 36 36 36 76 76 76 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 35 35 35 73 73 73 F F F 22 440-459 34 34 34 72 72 72 F F F 23 460-479 33 33 33 70 70 70 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 35 35 35 73 73 73 F F F 26 520-539 37 37 37 77 77 77 F F F 27 540-559 37 37 37 77 77 77 F F F 28 560-579 36 36 36 76 76 76 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 37, Max flush: 77 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.10673898Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.152234928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.152842663Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.152885292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.152900652Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.152915092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344} {"msg":"Crucible b5205a07-4ff0-4790-9fbb-db9e6d6a0e4b has session id: 41fa27db-a459-4ab1-b439-e1ceb61afb6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.153479547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Upstairs opts: Upstairs UUID: b5205a07-4ff0-4790-9fbb-db9e6d6a0e4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.153509437Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Crucible stats registered with UUID: b5205a07-4ff0-4790-9fbb-db9e6d6a0e4b","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.153524516Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:40.153537027Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154017252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154042092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344} {"msg":"b5205a07-4ff0-4790-9fbb-db9e6d6a0e4b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154116021Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154134521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"0","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154173211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"0","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154192101Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"1","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154303649Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"1","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154346699Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"2","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154375699Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"2","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154398979Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154425518Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154453158Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43928)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154480778Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:53561)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154665626Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38820)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.154702996Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.155100212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"1","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.15529106Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"0","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.15533858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"2","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.155753006Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"1","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.155852895Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"0","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156024794Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"2","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156339391Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.15645991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[0]R flush_numbers[0..12]: [72, 72, 77, 77, 74, 73, 67, 72, 62, 70, 77, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156671518Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[0]R generation[0..12]: [34, 34, 37, 37, 36, 35, 30, 34, 27, 33, 37, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156692288Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156707058Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[1]R flush_numbers[0..12]: [72, 72, 77, 77, 74, 73, 67, 72, 62, 70, 77, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156720498Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[1]R generation[0..12]: [34, 34, 37, 37, 36, 35, 30, 34, 27, 33, 37, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156733707Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156747447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[2]R flush_numbers[0..12]: [72, 72, 77, 77, 74, 73, 67, 72, 62, 70, 77, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156760717Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[2]R generation[0..12]: [34, 34, 37, 37, 36, 35, 30, 34, 27, 33, 37, 28]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156773827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156787427Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156800097Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156812987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156829567Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156848397Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"":"downstairs","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156861936Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156873886Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"0","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156887266Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"1","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156900276Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"client":"2","":"downstairs","session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156913416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156925356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"b5205a07-4ff0-4790-9fbb-db9e6d6a0e4b is now active with session: 41fa27db-a459-4ab1-b439-e1ceb61afb6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156937896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156949685Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344,"session_id":"41fa27db-a459-4ab1-b439-e1ceb61afb6d"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156961525Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344} {"msg":"Activated sub_volume b5205a07-4ff0-4790-9fbb-db9e6d6a0e4b","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:40.156972535Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 203 len 36864 data: 4 3 3 3 4 4 4 3 3 02/30 Read block 340 len 36864 03/30 Write block 303 len 28672 data: 6 6 4 4 2 3 3 04/30 Write block 307 len 16384 data: 3 4 4 3 05/30 Read block 108 len 4096 06/30 Read block 56 len 8192 07/30 Read block 359 len 12288 08/30 Read block 201 len 28672 09/30 Write block 281 len 12288 data: 7 7 6 10/30 Read block 140 len 24576 11/30 Read block 16 len 40960 12/30 Write block 110 len 36864 data: 3 4 3 4 4 3 3 5 4 13/30 Read block 576 len 24576 14/30 Read block 176 len 36864 15/30 Write block 220 len 24576 data: 7 7 7 6 6 4 16/30 Read block 249 len 36864 17/30 Write block 277 len 12288 data: 4 5 7 18/30 Write block 390 len 12288 data: 4 3 3 19/30 Read block 176 len 40960 20/30 Flush 21/30 Flush 22/30 Flush 23/30 Write block 515 len 32768 data: 4 4 4 2 4 3 4 3 24/30 Write block 119 len 20480 data: 4 5 6 6 7 25/30 Flush 26/30 Write block 250 len 8192 data: 2 4 27/30 Read block 568 len 12288 28/30 Read block 104 len 8192 29/30 Write block 420 len 28672 data: 6 5 4 3 2 2 3 30/30 Write block 444 len 20480 data: 4 4 4 5 5 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:40.177259222Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 34 34 34 72 72 72 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 37 37 37 77 77 77 F F F 4 080-099 36 36 36 74 74 74 F F F 5 100-119 38 38 38 81 81 81 F F F 6 120-139 38 38 38 81 81 81 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 38 38 38 78 78 78 F F F 11 220-239 38 38 38 78 78 78 F F F 12 240-259 34 34 34 71 71 71 T T T 13 260-279 38 38 38 78 78 78 F F F 14 280-299 38 38 38 78 78 78 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 36 36 36 76 76 76 F F F 19 380-399 38 38 38 78 78 78 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 35 35 35 73 73 73 T T T 22 440-459 34 34 34 72 72 72 T T T 23 460-479 33 33 33 70 70 70 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 38 38 38 81 81 81 F F F 26 520-539 38 38 38 81 81 81 F F F 27 540-559 37 37 37 77 77 77 F F F 28 560-579 36 36 36 76 76 76 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 38, Max flush: 81 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 34 34 34 72 72 72 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 37 37 37 77 77 77 F F F 4 080-099 36 36 36 74 74 74 F F F 5 100-119 38 38 38 81 81 81 F F F 6 120-139 38 38 38 81 81 81 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 38 38 38 78 78 78 F F F 11 220-239 38 38 38 78 78 78 F F F 12 240-259 39 39 39 82 82 82 F F F 13 260-279 38 38 38 78 78 78 F F F 14 280-299 38 38 38 78 78 78 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 36 36 36 76 76 76 F F F 19 380-399 38 38 38 78 78 78 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 39 39 39 82 82 82 F F F 22 440-459 39 39 39 82 82 82 F F F 23 460-479 33 33 33 70 70 70 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 38 38 38 81 81 81 F F F 26 520-539 38 38 38 81 81 81 F F F 27 540-559 37 37 37 77 77 77 F F F 28 560-579 36 36 36 76 76 76 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 39, Max flush: 82 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.407911313Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:44.45022411Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.450575927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.482006913Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.482648817Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.482690477Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.482706666Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.482724416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352} {"msg":"Crucible 30923498-b803-40b9-b8ca-5bb9ceb38aea has session id: 17b7ddbc-086e-4553-8ded-a6bb7cdcf10a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.4834231Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Upstairs opts: Upstairs UUID: 30923498-b803-40b9-b8ca-5bb9ceb38aea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.483452259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Crucible stats registered with UUID: 30923498-b803-40b9-b8ca-5bb9ceb38aea","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.483467519Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:44.483484009Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.483922675Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.483946495Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352} {"msg":"30923498-b803-40b9-b8ca-5bb9ceb38aea active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.483993315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484014625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"0","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484049774Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"0","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484075194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"1","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484102114Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"1","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484129144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"2","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484155983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"2","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484179913Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484205203Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484230863Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43484)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484256542Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37385)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484279432Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56347)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484418301Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.484990466Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"1","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.485029515Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"2","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.485059295Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"0","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48563602Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"1","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.485672359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"2","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.485749329Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"0","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486280094Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486384613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[0]R flush_numbers[0..12]: [72, 72, 77, 77, 74, 81, 81, 72, 62, 70, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486467292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[0]R generation[0..12]: [34, 34, 37, 37, 36, 38, 38, 34, 27, 33, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486489162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486504182Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[1]R flush_numbers[0..12]: [72, 72, 77, 77, 74, 81, 81, 72, 62, 70, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486518102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[1]R generation[0..12]: [34, 34, 37, 37, 36, 38, 38, 34, 27, 33, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486531702Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486549522Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[2]R flush_numbers[0..12]: [72, 72, 77, 77, 74, 81, 81, 72, 62, 70, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486564031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[2]R generation[0..12]: [34, 34, 37, 37, 36, 38, 38, 34, 27, 33, 38, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486577431Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486590971Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486603941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486616721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486629541Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:42:44.486642071Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"":"downstairs","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48665969Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486671581Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"0","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48668495Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"1","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48669775Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"client":"2","":"downstairs","session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48671058Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48672245Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"30923498-b803-40b9-b8ca-5bb9ceb38aea is now active with session: 17b7ddbc-086e-4553-8ded-a6bb7cdcf10a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48673503Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48674855Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352,"session_id":"17b7ddbc-086e-4553-8ded-a6bb7cdcf10a"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.48676082Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352} {"msg":"Activated sub_volume 30923498-b803-40b9-b8ca-5bb9ceb38aea","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:44.486771689Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 577 len 4096 data: 5 02/30 Flush 03/30 Write block 78 len 24576 data: 6 5 5 6 4 4 04/30 Write block 543 len 4096 data: 5 05/30 Read block 424 len 8192 06/30 Flush 07/30 Write block 214 len 24576 data: 5 2 5 6 7 7 08/30 Write block 234 len 20480 data: 6 7 4 4 5 09/30 Write block 464 len 32768 data: 6 7 7 6 6 5 6 6 10/30 Read block 587 len 12288 11/30 Read block 529 len 24576 12/30 Write block 267 len 8192 data: 3 3 13/30 Read block 261 len 40960 14/30 Write block 388 len 40960 data: 4 4 5 4 4 3 3 3 5 5 15/30 Flush 16/30 Write block 387 len 24576 data: 4 5 5 6 5 5 17/30 Read block 217 len 32768 18/30 Read block 400 len 16384 19/30 Read block 200 len 20480 20/30 Write block 220 len 32768 data: 8 8 8 7 7 5 3 4 21/30 Read block 73 len 12288 22/30 Read block 51 len 24576 23/30 Read block 24 len 24576 24/30 Write block 29 len 12288 data: 4 5 5 25/30 Write block 96 len 32768 data: 2 2 2 2 2 3 3 4 26/30 Write block 444 len 16384 data: 5 5 5 6 27/30 Read block 86 len 40960 28/30 Read block 410 len 32768 29/30 Write block 110 len 4096 data: 4 30/30 Write block 226 len 12288 data: 4 5 5 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 1 4 Done 14 13 14 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:44.50661525Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 34 34 34 72 72 72 T T T 2 040-059 37 37 37 77 77 77 F F F 3 060-079 40 40 40 84 84 84 F F F 4 080-099 40 40 40 84 84 84 T T T 5 100-119 38 38 38 81 81 81 T T T 6 120-139 38 38 38 81 81 81 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 40 40 40 85 85 85 F F F 11 220-239 40 40 40 85 85 85 T T T 12 240-259 39 39 39 82 82 82 F F F 13 260-279 40 40 40 85 85 85 F F F 14 280-299 38 38 38 78 78 78 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 36 36 36 76 76 76 F F F 19 380-399 40 40 40 85 85 85 T T T 20 400-419 31 31 31 68 68 68 F F F 21 420-439 39 39 39 82 82 82 F F F 22 440-459 39 39 39 82 82 82 T T T 23 460-479 40 40 40 85 85 85 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 38 38 38 81 81 81 F F F 26 520-539 38 38 38 81 81 81 F F F 27 540-559 40 40 40 84 84 84 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 40, Max flush: 85 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 41 41 41 86 86 86 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 40 40 40 84 84 84 F F F 4 080-099 41 41 41 86 86 86 F F F 5 100-119 41 41 41 86 86 86 F F F 6 120-139 38 38 38 81 81 81 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 27 27 27 62 62 62 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 40 40 40 85 85 85 F F F 11 220-239 41 41 41 86 86 86 F F F 12 240-259 39 39 39 82 82 82 F F F 13 260-279 40 40 40 85 85 85 F F F 14 280-299 38 38 38 78 78 78 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 36 36 36 76 76 76 F F F 19 380-399 41 41 41 86 86 86 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 39 39 39 82 82 82 F F F 22 440-459 41 41 41 86 86 86 F F F 23 460-479 40 40 40 85 85 85 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 38 38 38 81 81 81 F F F 26 520-539 38 38 38 81 81 81 F F F 27 540-559 40 40 40 84 84 84 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 41, Max flush: 86 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.779394189Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:48.821938524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.822332391Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.853046203Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.853619318Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.853653938Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.853667568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.853681887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360} {"msg":"Crucible 6c2d7282-5037-49d4-83ce-bf4aa6ab35c0 has session id: 9770534a-4385-4954-8b71-ee5fdf6185d8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854212072Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Upstairs opts: Upstairs UUID: 6c2d7282-5037-49d4-83ce-bf4aa6ab35c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854246762Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Crucible stats registered with UUID: 6c2d7282-5037-49d4-83ce-bf4aa6ab35c0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854263072Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:48.854276952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854730958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854754928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360} {"msg":"6c2d7282-5037-49d4-83ce-bf4aa6ab35c0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854825477Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854843787Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"0","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854877106Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"0","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854901686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"1","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854933406Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"1","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.854962456Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"2","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.855053365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"2","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.855136254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.855185754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.855223313Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:39027)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.855278403Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47450)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.855316982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47305)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.855353982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.856012976Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"2","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.856052806Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"1","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.856136615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"0","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.85667815Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"2","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.85671686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"1","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.856861979Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"0","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857266935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857390774Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[0]R flush_numbers[0..12]: [72, 86, 77, 84, 86, 86, 81, 72, 62, 70, 85, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857522993Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[0]R generation[0..12]: [34, 41, 37, 40, 41, 41, 38, 34, 27, 33, 40, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857543233Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857558112Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[1]R flush_numbers[0..12]: [72, 86, 77, 84, 86, 86, 81, 72, 62, 70, 85, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857572132Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[1]R generation[0..12]: [34, 41, 37, 40, 41, 41, 38, 34, 27, 33, 40, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857585652Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857599212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[2]R flush_numbers[0..12]: [72, 86, 77, 84, 86, 86, 81, 72, 62, 70, 85, 86]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857612592Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[2]R generation[0..12]: [34, 41, 37, 40, 41, 41, 38, 34, 27, 33, 40, 41]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857625802Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857639272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857652032Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857664791Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 87","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857687611Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857700881Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"":"downstairs","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857716031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857737911Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"0","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.85776206Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"1","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.8577767Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"client":"2","":"downstairs","session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.85779576Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.85781772Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"6c2d7282-5037-49d4-83ce-bf4aa6ab35c0 is now active with session: 9770534a-4385-4954-8b71-ee5fdf6185d8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.85784224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.85786461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :1360,"session_id":"9770534a-4385-4954-8b71-ee5fdf6185d8"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857893249Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360} {"msg":"Activated sub_volume 6c2d7282-5037-49d4-83ce-bf4aa6ab35c0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:48.857910779Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 278 len 28672 data: 6 8 7 8 8 7 6 02/30 Read block 0 len 28672 03/30 Read block 336 len 28672 04/30 Read block 228 len 4096 05/30 Read block 7 len 16384 06/30 Read block 223 len 16384 07/30 Write block 269 len 32768 data: 3 3 3 3 3 3 2 2 08/30 Write block 233 len 16384 data: 6 7 8 5 09/30 Read block 539 len 16384 10/30 Read block 358 len 24576 11/30 Write block 166 len 4096 data: 2 12/30 Read block 387 len 24576 13/30 Write block 134 len 20480 data: 2 2 3 4 3 14/30 Read block 407 len 12288 15/30 Read block 236 len 20480 16/30 Read block 550 len 20480 17/30 Read block 144 len 24576 18/30 Flush 19/30 Flush 20/30 Write block 422 len 28672 data: 5 4 3 3 4 3 5 21/30 Write block 214 len 32768 data: 6 3 6 7 8 8 9 9 22/30 Flush 23/30 Write block 269 len 8192 data: 4 4 24/30 Read block 2 len 24576 25/30 Read block 397 len 24576 26/30 Flush 27/30 Write block 246 len 32768 data: 4 3 3 3 3 5 4 4 28/30 Read block 22 len 28672 29/30 Read block 529 len 36864 30/30 Write block 80 len 8192 data: 6 7 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:48.878728011Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 41 41 41 86 86 86 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 40 40 40 84 84 84 F F F 4 080-099 41 41 41 86 86 86 T T T 5 100-119 41 41 41 86 86 86 F F F 6 120-139 42 42 42 87 87 87 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 42 42 42 87 87 87 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 42 42 42 89 89 89 F F F 11 220-239 42 42 42 89 89 89 F F F 12 240-259 39 39 39 82 82 82 T T T 13 260-279 42 42 42 90 90 90 F F F 14 280-299 42 42 42 87 87 87 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 36 36 36 76 76 76 F F F 19 380-399 41 41 41 86 86 86 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 42 42 42 89 89 89 F F F 22 440-459 41 41 41 86 86 86 F F F 23 460-479 40 40 40 85 85 85 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 38 38 38 81 81 81 F F F 26 520-539 38 38 38 81 81 81 F F F 27 540-559 40 40 40 84 84 84 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 42, Max flush: 90 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 41 41 41 86 86 86 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 40 40 40 84 84 84 F F F 4 080-099 43 43 43 91 91 91 F F F 5 100-119 41 41 41 86 86 86 F F F 6 120-139 42 42 42 87 87 87 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 42 42 42 87 87 87 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 42 42 42 89 89 89 F F F 11 220-239 42 42 42 89 89 89 F F F 12 240-259 43 43 43 91 91 91 F F F 13 260-279 42 42 42 90 90 90 F F F 14 280-299 42 42 42 87 87 87 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 21 21 21 44 44 44 F F F 18 360-379 36 36 36 76 76 76 F F F 19 380-399 41 41 41 86 86 86 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 42 42 42 89 89 89 F F F 22 440-459 41 41 41 86 86 86 F F F 23 460-479 40 40 40 85 85 85 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 38 38 38 81 81 81 F F F 26 520-539 38 38 38 81 81 81 F F F 27 540-559 40 40 40 84 84 84 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 43, Max flush: 91 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.131541122Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:53.175039868Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.175413205Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.206670602Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.207337316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.207379305Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.207396345Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.207414065Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368} {"msg":"Crucible 6de35754-3c12-4730-9047-f5bb1237512d has session id: 9be37a44-4c05-4248-916d-e815c326867a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208049439Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Upstairs opts: Upstairs UUID: 6de35754-3c12-4730-9047-f5bb1237512d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208129839Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Crucible stats registered with UUID: 6de35754-3c12-4730-9047-f5bb1237512d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208155278Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:53.208168578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208628774Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208662104Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368} {"msg":"6de35754-3c12-4730-9047-f5bb1237512d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208714583Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208740913Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"0","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208771403Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"0","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208786073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"1","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208801943Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"1","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208815523Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"2","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208829502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"2","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208842252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208855702Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.208869112Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64137)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.209037141Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44351)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.20907728Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53736)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.20910776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.209682604Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"1","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.209717364Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"2","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.209870573Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"0","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.210375528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"1","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.210411198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"0","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.210433488Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"2","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211083952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211110972Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[0]R flush_numbers[0..12]: [72, 86, 77, 84, 91, 86, 87, 72, 87, 70, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211141021Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[0]R generation[0..12]: [34, 41, 37, 40, 43, 41, 42, 34, 42, 33, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211157211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211171671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[1]R flush_numbers[0..12]: [72, 86, 77, 84, 91, 86, 87, 72, 87, 70, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211185481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[1]R generation[0..12]: [34, 41, 37, 40, 43, 41, 42, 34, 42, 33, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211198901Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211212411Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[2]R flush_numbers[0..12]: [72, 86, 77, 84, 91, 86, 87, 72, 87, 70, 89, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.21122895Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[2]R generation[0..12]: [34, 41, 37, 40, 43, 41, 42, 34, 42, 33, 42, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211242461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.21125598Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.21126845Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.21128123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.21129366Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.21130637Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"":"downstairs","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.21131939Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.21133092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"0","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211343709Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"1","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211356489Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"client":"2","":"downstairs","session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211369429Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:42:53.211381309Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"6de35754-3c12-4730-9047-f5bb1237512d is now active with session: 9be37a44-4c05-4248-916d-e815c326867a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211397889Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211409699Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368,"session_id":"9be37a44-4c05-4248-916d-e815c326867a"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211421459Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368} {"msg":"Activated sub_volume 6de35754-3c12-4730-9047-f5bb1237512d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:53.211434489Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 101 len 32768 data: 4 4 5 3 3 3 2 3 02/30 Read block 482 len 24576 03/30 Write block 387 len 4096 data: 5 04/30 Flush 05/30 Write block 474 len 20480 data: 4 4 4 5 5 06/30 Write block 471 len 12288 data: 7 5 4 07/30 Read block 136 len 8192 08/30 Read block 262 len 8192 09/30 Read block 507 len 8192 10/30 Write block 340 len 16384 data: 5 4 4 4 11/30 Write block 371 len 40960 data: 6 6 6 7 7 6 5 5 4 4 12/30 Write block 91 len 40960 data: 6 6 5 5 4 3 3 3 3 3 13/30 Read block 508 len 40960 14/30 Flush 15/30 Flush 16/30 Write block 464 len 32768 data: 7 8 8 7 7 6 7 8 17/30 Write block 388 len 40960 data: 6 6 7 6 6 4 4 4 6 6 18/30 Read block 231 len 24576 19/30 Read block 250 len 40960 20/30 Read block 90 len 28672 21/30 Read block 373 len 12288 22/30 Read block 432 len 8192 23/30 Write block 544 len 28672 data: 4 5 6 6 6 7 5 24/30 Read block 398 len 12288 25/30 Read block 317 len 40960 26/30 Flush 27/30 Write block 262 len 28672 data: 3 3 5 4 3 4 4 28/30 Write block 26 len 12288 data: 3 3 4 29/30 Write block 207 len 28672 data: 5 5 5 4 4 4 4 30/30 Read block 507 len 12288 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 4 2 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:53.232172541Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 41 41 41 86 86 86 T T T 2 040-059 37 37 37 77 77 77 F F F 3 060-079 40 40 40 84 84 84 F F F 4 080-099 44 44 44 93 93 93 F F F 5 100-119 44 44 44 93 93 93 F F F 6 120-139 42 42 42 87 87 87 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 42 42 42 87 87 87 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 42 42 42 89 89 89 T T T 11 220-239 42 42 42 89 89 89 F F F 12 240-259 43 43 43 91 91 91 F F F 13 260-279 42 42 42 90 90 90 T T T 14 280-299 42 42 42 87 87 87 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 44 44 44 93 93 93 F F F 18 360-379 44 44 44 93 93 93 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 42 42 42 89 89 89 F F F 22 440-459 41 41 41 86 86 86 F F F 23 460-479 44 44 44 95 95 95 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 38 38 38 81 81 81 F F F 26 520-539 38 38 38 81 81 81 F F F 27 540-559 44 44 44 95 95 95 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 44, Max flush: 95 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 45 45 45 96 96 96 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 40 40 40 84 84 84 F F F 4 080-099 44 44 44 93 93 93 F F F 5 100-119 44 44 44 93 93 93 F F F 6 120-139 42 42 42 87 87 87 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 42 42 42 87 87 87 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 45 45 45 96 96 96 F F F 11 220-239 42 42 42 89 89 89 F F F 12 240-259 43 43 43 91 91 91 F F F 13 260-279 45 45 45 96 96 96 F F F 14 280-299 42 42 42 87 87 87 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 36 36 36 76 76 76 F F F 17 340-359 44 44 44 93 93 93 F F F 18 360-379 44 44 44 93 93 93 F F F 19 380-399 44 44 44 95 95 95 F F F 20 400-419 31 31 31 68 68 68 F F F 21 420-439 42 42 42 89 89 89 F F F 22 440-459 41 41 41 86 86 86 F F F 23 460-479 44 44 44 95 95 95 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 38 38 38 81 81 81 F F F 26 520-539 38 38 38 81 81 81 F F F 27 540-559 44 44 44 95 95 95 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 45, Max flush: 96 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.490246964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.534945809Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.535573104Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.535623594Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.535640993Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.535658533Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376} {"msg":"Crucible 4eaa7ee3-6fb0-412e-8013-de230a3047ff has session id: 460df94c-764c-4d0f-86c3-d3fd05c35da5","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536238108Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Upstairs opts: Upstairs UUID: 4eaa7ee3-6fb0-412e-8013-de230a3047ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536269937Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Crucible stats registered with UUID: 4eaa7ee3-6fb0-412e-8013-de230a3047ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536291447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:57.536314017Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536761653Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536794123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376} {"msg":"4eaa7ee3-6fb0-412e-8013-de230a3047ff active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536848902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536876202Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"0","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536908452Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"0","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536926872Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"1","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536953171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"1","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.536978631Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"2","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.537005281Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"2","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.537030271Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.53705305Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.53708507Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"ds_connection connected from Ok(127.0.0.1:64527)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.5371119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46381)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.53713653Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"ds_connection connected from Ok(127.0.0.1:56589)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.537163829Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.537744804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"0","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.537880933Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"1","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.538103251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"2","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.538431288Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"1","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.538564727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"0","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.538847774Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"2","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539054642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539177261Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[0]R flush_numbers[0..12]: [72, 96, 77, 84, 93, 93, 87, 72, 87, 70, 96, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539490308Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[0]R generation[0..12]: [34, 45, 37, 40, 44, 44, 42, 34, 42, 33, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539513988Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539533168Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[1]R flush_numbers[0..12]: [72, 96, 77, 84, 93, 93, 87, 72, 87, 70, 96, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539547378Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[1]R generation[0..12]: [34, 45, 37, 40, 44, 44, 42, 34, 42, 33, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539560998Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539574708Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[2]R flush_numbers[0..12]: [72, 96, 77, 84, 93, 93, 87, 72, 87, 70, 96, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539587867Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[2]R generation[0..12]: [34, 45, 37, 40, 44, 44, 42, 34, 42, 33, 45, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539601147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539614747Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539627447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539640237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:42:57.539652757Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539668917Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"":"downstairs","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539681987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539693747Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"0","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539706677Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"1","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539719456Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"client":"2","":"downstairs","session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539734456Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539746596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"4eaa7ee3-6fb0-412e-8013-de230a3047ff is now active with session: 460df94c-764c-4d0f-86c3-d3fd05c35da5","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539759276Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539770906Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376,"session_id":"460df94c-764c-4d0f-86c3-d3fd05c35da5"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539782876Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376} {"msg":"Activated sub_volume 4eaa7ee3-6fb0-412e-8013-de230a3047ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:42:57.539793866Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 516 len 32768 data: 5 5 3 5 4 5 4 3 02/30 Write block 68 len 16384 data: 4 4 3 4 03/30 Write block 267 len 36864 data: 5 5 5 5 4 4 4 4 3 04/30 Write block 98 len 24576 data: 4 4 4 5 5 6 05/30 Write block 203 len 8192 data: 5 4 06/30 Read block 238 len 32768 07/30 Read block 500 len 8192 08/30 Flush 09/30 Write block 27 len 24576 data: 4 5 5 6 6 6 10/30 Read block 304 len 24576 11/30 Write block 290 len 32768 data: 4 4 3 3 4 4 4 3 12/30 Read block 519 len 12288 13/30 Write block 399 len 36864 data: 5 6 7 7 7 5 3 3 2 14/30 Read block 488 len 40960 15/30 Read block 296 len 8192 16/30 Read block 140 len 32768 17/30 Write block 452 len 4096 data: 6 18/30 Read block 482 len 32768 19/30 Flush 20/30 Write block 204 len 8192 data: 5 4 21/30 Write block 335 len 20480 data: 6 6 6 6 5 22/30 Read block 138 len 8192 23/30 Flush 24/30 Flush 25/30 Write block 411 len 4096 data: 3 26/30 Read block 448 len 8192 27/30 Read block 71 len 24576 28/30 Write block 178 len 4096 data: 3 29/30 Read block 184 len 4096 30/30 Read block 534 len 4096 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:42:57.559057611Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 46 46 46 98 98 98 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 46 46 46 97 97 97 F F F 4 080-099 46 46 46 97 97 97 F F F 5 100-119 46 46 46 97 97 97 F F F 6 120-139 42 42 42 87 87 87 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 42 42 42 87 87 87 T T T 9 180-199 33 33 33 70 70 70 F F F 10 200-219 46 46 46 99 99 99 F F F 11 220-239 42 42 42 89 89 89 F F F 12 240-259 43 43 43 91 91 91 F F F 13 260-279 46 46 46 97 97 97 F F F 14 280-299 46 46 46 98 98 98 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 46 46 46 99 99 99 F F F 17 340-359 44 44 44 93 93 93 F F F 18 360-379 44 44 44 93 93 93 F F F 19 380-399 46 46 46 98 98 98 F F F 20 400-419 46 46 46 98 98 98 T T T 21 420-439 42 42 42 89 89 89 F F F 22 440-459 46 46 46 98 98 98 F F F 23 460-479 44 44 44 95 95 95 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 46 46 46 97 97 97 F F F 26 520-539 46 46 46 97 97 97 F F F 27 540-559 44 44 44 95 95 95 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 46, Max flush: 99 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 F F F 1 020-039 46 46 46 98 98 98 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 46 46 46 97 97 97 F F F 4 080-099 46 46 46 97 97 97 F F F 5 100-119 46 46 46 97 97 97 F F F 6 120-139 42 42 42 87 87 87 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 47 47 47 100 100 100 F F F 9 180-199 33 33 33 70 70 70 F F F 10 200-219 46 46 46 99 99 99 F F F 11 220-239 42 42 42 89 89 89 F F F 12 240-259 43 43 43 91 91 91 F F F 13 260-279 46 46 46 97 97 97 F F F 14 280-299 46 46 46 98 98 98 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 46 46 46 99 99 99 F F F 17 340-359 44 44 44 93 93 93 F F F 18 360-379 44 44 44 93 93 93 F F F 19 380-399 46 46 46 98 98 98 F F F 20 400-419 47 47 47 100 100 100 F F F 21 420-439 42 42 42 89 89 89 F F F 22 440-459 46 46 46 98 98 98 F F F 23 460-479 44 44 44 95 95 95 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 46 46 46 97 97 97 F F F 26 520-539 46 46 46 97 97 97 F F F 27 540-559 44 44 44 95 95 95 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 31 31 31 68 68 68 F F F Max gen: 47, Max flush: 100 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.784142743Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.828516861Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.829203265Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.829258405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.829277144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.829295004Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384} {"msg":"Crucible 85fee693-2d65-4b2a-9fd6-5bb00361df78 has session id: cc234d59-6bc5-488b-bfe0-aff6aabab582","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.829677321Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Upstairs opts: Upstairs UUID: 85fee693-2d65-4b2a-9fd6-5bb00361df78, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.82972662Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Crucible stats registered with UUID: 85fee693-2d65-4b2a-9fd6-5bb00361df78","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.82975852Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:01.82978869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830270055Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830296835Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384} {"msg":"85fee693-2d65-4b2a-9fd6-5bb00361df78 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830336395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830352205Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"0","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830380544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"0","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830401224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"1","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830431914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"1","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830510413Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"2","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830558193Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"2","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830587102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830613462Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830639592Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"ds_connection connected from Ok(127.0.0.1:52149)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830768521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"ds_connection connected from Ok(127.0.0.1:39377)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.830809751Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"ds_connection connected from Ok(127.0.0.1:42431)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.83082756Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.831434265Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"2","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.831463865Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"1","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.831480714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"0","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.832208998Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"0","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.832239718Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"2","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.832256177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"1","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.832782103Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.832884162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[0]R flush_numbers[0..12]: [72, 98, 77, 97, 97, 97, 87, 72, 100, 70, 99, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.832976401Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[0]R generation[0..12]: [34, 46, 37, 46, 46, 46, 42, 34, 47, 33, 46, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.832997091Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833011811Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[1]R flush_numbers[0..12]: [72, 98, 77, 97, 97, 97, 87, 72, 100, 70, 99, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833025431Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[1]R generation[0..12]: [34, 46, 37, 46, 46, 46, 42, 34, 47, 33, 46, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.83304226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833067Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[2]R flush_numbers[0..12]: [72, 98, 77, 97, 97, 97, 87, 72, 100, 70, 99, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.83309367Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[2]R generation[0..12]: [34, 46, 37, 46, 46, 46, 42, 34, 47, 33, 46, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.83311345Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833128219Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833150779Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833174849Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833199709Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833221249Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"":"downstairs","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833234869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833253448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"0","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833282118Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"1","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833307678Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"client":"2","":"downstairs","session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833327078Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833339568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"85fee693-2d65-4b2a-9fd6-5bb00361df78 is now active with session: cc234d59-6bc5-488b-bfe0-aff6aabab582","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833355857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833376877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384,"session_id":"cc234d59-6bc5-488b-bfe0-aff6aabab582"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833399557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384} {"msg":"Activated sub_volume 85fee693-2d65-4b2a-9fd6-5bb00361df78","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:01.833420437Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 36864 data: 4 3 3 3 2 2 2 2 2 02/30 Write block 345 len 36864 data: 3 2 2 2 2 3 3 3 2 03/30 Read block 193 len 12288 04/30 Write block 411 len 24576 data: 4 4 4 4 4 6 05/30 Write block 592 len 20480 data: 3 3 3 3 3 06/30 Read block 31 len 16384 07/30 Write block 543 len 36864 data: 6 5 6 7 7 7 8 6 5 08/30 Write block 88 len 24576 data: 4 6 6 7 7 6 09/30 Read block 356 len 36864 10/30 Read block 99 len 20480 11/30 Write block 254 len 16384 data: 4 5 5 6 12/30 Read block 66 len 20480 13/30 Write block 234 len 36864 data: 8 9 6 5 6 5 8 7 6 14/30 Flush 15/30 Write block 36 len 12288 data: 6 7 5 16/30 Read block 429 len 4096 17/30 Write block 364 len 28672 data: 4 3 4 3 3 4 4 18/30 Read block 240 len 40960 19/30 Read block 568 len 16384 20/30 Write block 391 len 32768 data: 7 7 5 5 5 7 7 5 21/30 Write block 60 len 40960 data: 5 6 7 6 6 5 4 4 5 5 22/30 Write block 7 len 8192 data: 7 6 23/30 Read block 462 len 4096 24/30 Write block 514 len 20480 data: 5 5 6 6 4 25/30 Read block 256 len 36864 26/30 Write block 134 len 8192 data: 3 3 27/30 Read block 589 len 24576 28/30 Read block 200 len 12288 29/30 Write block 506 len 8192 data: 3 5 30/30 Read block 548 len 36864 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 3 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:01.853911391Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 72 72 72 T T T 1 020-039 46 46 46 98 98 98 T T T 2 040-059 37 37 37 77 77 77 F F F 3 060-079 46 46 46 97 97 97 T T T 4 080-099 48 48 48 101 101 101 F F F 5 100-119 46 46 46 97 97 97 F F F 6 120-139 42 42 42 87 87 87 T T T 7 140-159 34 34 34 72 72 72 F F F 8 160-179 48 48 48 101 101 101 F F F 9 180-199 48 48 48 101 101 101 F F F 10 200-219 46 46 46 99 99 99 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 48 48 48 101 101 101 F F F 13 260-279 46 46 46 97 97 97 F F F 14 280-299 46 46 46 98 98 98 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 46 46 46 99 99 99 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 44 44 44 93 93 93 T T T 19 380-399 46 46 46 98 98 98 T T T 20 400-419 48 48 48 101 101 101 F F F 21 420-439 42 42 42 89 89 89 F F F 22 440-459 46 46 46 98 98 98 F F F 23 460-479 44 44 44 95 95 95 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 46 46 46 97 97 97 T T T 26 520-539 46 46 46 97 97 97 F F F 27 540-559 48 48 48 101 101 101 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 48 48 48 101 101 101 F F F Max gen: 48, Max flush: 101 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 102 102 102 F F F 1 020-039 49 49 49 102 102 102 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 49 49 49 102 102 102 F F F 4 080-099 48 48 48 101 101 101 F F F 5 100-119 46 46 46 97 97 97 F F F 6 120-139 49 49 49 102 102 102 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 48 48 48 101 101 101 F F F 9 180-199 48 48 48 101 101 101 F F F 10 200-219 46 46 46 99 99 99 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 48 48 48 101 101 101 F F F 13 260-279 46 46 46 97 97 97 F F F 14 280-299 46 46 46 98 98 98 F F F 15 300-319 38 38 38 78 78 78 F F F 16 320-339 46 46 46 99 99 99 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 49 49 49 102 102 102 F F F 19 380-399 49 49 49 102 102 102 F F F 20 400-419 48 48 48 101 101 101 F F F 21 420-439 42 42 42 89 89 89 F F F 22 440-459 46 46 46 98 98 98 F F F 23 460-479 44 44 44 95 95 95 F F F 24 480-499 33 33 33 70 70 70 F F F 25 500-519 49 49 49 102 102 102 F F F 26 520-539 46 46 46 97 97 97 F F F 27 540-559 48 48 48 101 101 101 F F F 28 560-579 40 40 40 83 83 83 F F F 29 580-599 48 48 48 101 101 101 F F F Max gen: 49, Max flush: 102 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.1057693Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.150862283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.151511457Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.151555116Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.151570546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.151585206Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392} {"msg":"Crucible f620df33-561b-4a75-84df-6e75408a4a86 has session id: 607f2856-81bd-466c-b4c7-704079db6474","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.15219006Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Upstairs opts: Upstairs UUID: f620df33-561b-4a75-84df-6e75408a4a86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.15223046Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Crucible stats registered with UUID: f620df33-561b-4a75-84df-6e75408a4a86","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.15225166Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:06.15226488Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152697306Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152718076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392} {"msg":"f620df33-561b-4a75-84df-6e75408a4a86 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152786445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152803335Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"0","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152832215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"0","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152846785Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"1","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152860905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"1","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152875004Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"2","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152888864Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"2","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152901534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152923994Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152949274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"ds_connection connected from Ok(127.0.0.1:55355)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.152975313Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"ds_connection connected from Ok(127.0.0.1:57005)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.153007703Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"ds_connection connected from Ok(127.0.0.1:33064)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.153034343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.153697677Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"0","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.153740446Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"2","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.153881885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"1","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.1544296Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"0","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.15447212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"2","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.15450753Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"1","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155069705Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155094454Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[0]R flush_numbers[0..12]: [102, 102, 77, 102, 101, 97, 102, 72, 101, 101, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155177684Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[0]R generation[0..12]: [49, 49, 37, 49, 48, 46, 49, 34, 48, 48, 46, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155197823Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155212103Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[1]R flush_numbers[0..12]: [102, 102, 77, 102, 101, 97, 102, 72, 101, 101, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155225743Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[1]R generation[0..12]: [49, 49, 37, 49, 48, 46, 49, 34, 48, 48, 46, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155243123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155257183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[2]R flush_numbers[0..12]: [102, 102, 77, 102, 101, 97, 102, 72, 101, 101, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155270793Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[2]R generation[0..12]: [49, 49, 37, 49, 48, 46, 49, 34, 48, 48, 46, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155283912Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155297303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155309782Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155322622Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155335072Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155347672Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"":"downstairs","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155360292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155371982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"0","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155384882Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"1","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:43:06.155397511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"client":"2","":"downstairs","session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155416561Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155428631Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"f620df33-561b-4a75-84df-6e75408a4a86 is now active with session: 607f2856-81bd-466c-b4c7-704079db6474","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155441221Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155454931Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392,"session_id":"607f2856-81bd-466c-b4c7-704079db6474"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155467071Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392} {"msg":"Activated sub_volume f620df33-561b-4a75-84df-6e75408a4a86","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:06.155478171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 425 len 32768 data: 4 5 4 6 5 5 5 4 02/30 Write block 467 len 8192 data: 8 8 03/30 Write block 462 len 20480 data: 5 6 8 9 9 04/30 Write block 61 len 32768 data: 7 8 7 7 6 5 5 6 05/30 Write block 393 len 20480 data: 6 6 6 8 8 06/30 Read block 325 len 4096 07/30 Write block 376 len 16384 data: 7 6 6 5 08/30 Read block 427 len 24576 09/30 Read block 393 len 24576 10/30 Flush 11/30 Write block 567 len 20480 data: 4 4 3 3 4 12/30 Write block 412 len 36864 data: 5 5 5 5 7 6 6 6 7 13/30 Write block 271 len 40960 data: 5 5 5 5 4 3 5 7 9 8 14/30 Write block 120 len 40960 data: 6 7 7 8 6 4 4 4 3 3 15/30 Read block 408 len 24576 16/30 Write block 181 len 4096 data: 4 17/30 Write block 182 len 36864 data: 4 3 3 3 3 3 2 2 2 18/30 Read block 11 len 12288 19/30 Write block 316 len 28672 data: 4 4 5 5 5 6 4 20/30 Write block 264 len 32768 data: 6 5 4 6 6 6 6 6 21/30 Write block 442 len 12288 data: 4 3 6 22/30 Read block 455 len 28672 23/30 Read block 138 len 28672 24/30 Write block 325 len 4096 data: 5 25/30 Write block 510 len 28672 data: 4 4 4 5 6 6 7 26/30 Read block 386 len 24576 27/30 Write block 532 len 20480 data: 3 4 5 6 5 28/30 Write block 192 len 12288 data: 3 3 3 29/30 Write block 486 len 20480 data: 5 4 3 4 4 30/30 Write block 91 len 20480 data: 8 8 7 6 5 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 5 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:06.173726136Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 102 102 102 F F F 1 020-039 49 49 49 102 102 102 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 50 50 50 103 103 103 F F F 4 080-099 48 48 48 101 101 101 T T T 5 100-119 46 46 46 97 97 97 F F F 6 120-139 49 49 49 102 102 102 T T T 7 140-159 34 34 34 72 72 72 F F F 8 160-179 48 48 48 101 101 101 F F F 9 180-199 48 48 48 101 101 101 T T T 10 200-219 46 46 46 99 99 99 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 48 48 48 101 101 101 F F F 13 260-279 46 46 46 97 97 97 T T T 14 280-299 46 46 46 98 98 98 T T T 15 300-319 38 38 38 78 78 78 T T T 16 320-339 46 46 46 99 99 99 T T T 17 340-359 48 48 48 101 101 101 F F F 18 360-379 50 50 50 103 103 103 F F F 19 380-399 50 50 50 103 103 103 F F F 20 400-419 48 48 48 101 101 101 T T T 21 420-439 50 50 50 103 103 103 T T T 22 440-459 46 46 46 98 98 98 T T T 23 460-479 50 50 50 103 103 103 F F F 24 480-499 33 33 33 70 70 70 T T T 25 500-519 49 49 49 102 102 102 T T T 26 520-539 46 46 46 97 97 97 T T T 27 540-559 48 48 48 101 101 101 F F F 28 560-579 40 40 40 83 83 83 T T T 29 580-599 48 48 48 101 101 101 F F F Max gen: 50, Max flush: 103 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 102 102 102 F F F 1 020-039 49 49 49 102 102 102 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 50 50 50 103 103 103 F F F 4 080-099 51 51 51 104 104 104 F F F 5 100-119 46 46 46 97 97 97 F F F 6 120-139 51 51 51 104 104 104 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 48 48 48 101 101 101 F F F 9 180-199 51 51 51 104 104 104 F F F 10 200-219 46 46 46 99 99 99 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 48 48 48 101 101 101 F F F 13 260-279 51 51 51 104 104 104 F F F 14 280-299 51 51 51 104 104 104 F F F 15 300-319 51 51 51 104 104 104 F F F 16 320-339 51 51 51 104 104 104 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 50 50 50 103 103 103 F F F 19 380-399 50 50 50 103 103 103 F F F 20 400-419 51 51 51 104 104 104 F F F 21 420-439 51 51 51 104 104 104 F F F 22 440-459 51 51 51 104 104 104 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 51 51 51 104 104 104 F F F 25 500-519 51 51 51 104 104 104 F F F 26 520-539 51 51 51 104 104 104 F F F 27 540-559 48 48 48 101 101 101 F F F 28 560-579 51 51 51 104 104 104 F F F 29 580-599 48 48 48 101 101 101 F F F Max gen: 51, Max flush: 104 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.454104987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.498465296Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.49906843Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.49910973Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.49912386Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.49913805Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400} {"msg":"Crucible 5ee1d4de-852f-4638-894e-e28eddf2d0a0 has session id: 0fa85747-1f6c-4f26-92fd-96e08b01f5e6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.499666375Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Upstairs opts: Upstairs UUID: 5ee1d4de-852f-4638-894e-e28eddf2d0a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.499707704Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Crucible stats registered with UUID: 5ee1d4de-852f-4638-894e-e28eddf2d0a0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.499732524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:20.499753344Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.50022784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.50025486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400} {"msg":"5ee1d4de-852f-4638-894e-e28eddf2d0a0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500318449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500337529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"0","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500371018Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"0","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500386289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"1","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500400168Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"1","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500413118Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"2","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500427028Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"2","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500439958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500453468Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500466598Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:33675)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500561937Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:65159)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500602416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:40837)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.500629956Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.50127814Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"0","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.50130728Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"2","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.501814105Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"1","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.501926155Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"2","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.501951264Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"0","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.50243057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"1","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.502571589Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.502607578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[0]R flush_numbers[0..12]: [102, 102, 77, 103, 104, 97, 104, 72, 101, 104, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.502969855Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[0]R generation[0..12]: [49, 49, 37, 50, 51, 46, 51, 34, 48, 51, 46, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503005765Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503032834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[1]R flush_numbers[0..12]: [102, 102, 77, 103, 104, 97, 104, 72, 101, 104, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503058884Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[1]R generation[0..12]: [49, 49, 37, 50, 51, 46, 51, 34, 48, 51, 46, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503083444Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503109334Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[2]R flush_numbers[0..12]: [102, 102, 77, 103, 104, 97, 104, 72, 101, 104, 99, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503137784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[2]R generation[0..12]: [49, 49, 37, 50, 51, 46, 51, 34, 48, 51, 46, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503166053Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503194193Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503229763Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503257532Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503283222Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503308882Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"":"downstairs","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503335102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503359542Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"0","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503386721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 client":"1","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503417611Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"client":"2","":"downstairs","session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503443601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503462211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"5ee1d4de-852f-4638-894e-e28eddf2d0a0 is now active with session: 0fa85747-1f6c-4f26-92fd-96e08b01f5e6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.503475421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.50348756Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400,"session_id":"0fa85747-1f6c-4f26-92fd-96e08b01f5e6"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.50350008Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400} {"msg":"Activated sub_volume 5ee1d4de-852f-4638-894e-e28eddf2d0a0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:20.50351115Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 213 len 28672 data: 5 7 4 7 8 9 9 02/30 Read block 337 len 40960 03/30 Write block 207 len 24576 data: 6 6 6 5 5 5 04/30 Read block 125 len 40960 05/30 Read block 74 len 24576 06/30 Write block 516 len 24576 data: 8 7 5 6 5 6 07/30 Write block 105 len 28672 data: 4 4 3 4 3 5 5 08/30 Read block 532 len 32768 09/30 Write block 307 len 40960 data: 4 5 5 4 3 3 3 3 3 5 10/30 Read block 389 len 36864 11/30 Flush 12/30 Write block 77 len 40960 data: 7 7 6 7 8 5 5 4 3 4 13/30 Read block 11 len 40960 14/30 Write block 520 len 4096 data: 6 15/30 Write block 521 len 36864 data: 7 5 4 3 4 5 6 5 5 16/30 Read block 466 len 4096 17/30 Write block 377 len 36864 data: 7 7 6 5 3 4 4 3 4 18/30 Read block 12 len 24576 19/30 Read block 598 len 4096 20/30 Read block 328 len 12288 21/30 Read block 149 len 40960 22/30 Read block 449 len 8192 23/30 Write block 517 len 12288 data: 8 6 7 24/30 Read block 424 len 40960 25/30 Write block 177 len 12288 data: 4 4 5 26/30 Write block 105 len 36864 data: 5 5 4 5 4 6 6 4 5 27/30 Write block 196 len 28672 data: 4 4 5 5 5 4 4 28/30 Read block 530 len 8192 29/30 Write block 96 len 28672 data: 4 4 5 5 5 6 6 30/30 Read block 442 len 32768 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 10 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 18 18 19 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:20.526472613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 102 102 102 F F F 1 020-039 49 49 49 102 102 102 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 50 50 50 103 103 103 T T T 4 080-099 51 51 51 104 104 104 T T T 5 100-119 52 52 52 105 105 105 T T T 6 120-139 51 51 51 104 104 104 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 48 48 48 101 101 101 T T T 9 180-199 51 51 51 104 104 104 T T T 10 200-219 52 52 52 105 105 105 T T T 11 220-239 48 48 48 101 101 101 F F F 12 240-259 48 48 48 101 101 101 F F F 13 260-279 51 51 51 104 104 104 F F F 14 280-299 51 51 51 104 104 104 F F F 15 300-319 52 52 52 105 105 105 F F F 16 320-339 51 51 51 104 104 104 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 50 50 50 103 103 103 T T T 19 380-399 50 50 50 103 103 103 T T T 20 400-419 51 51 51 104 104 104 F F F 21 420-439 51 51 51 104 104 104 F F F 22 440-459 51 51 51 104 104 104 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 51 51 51 104 104 104 F F F 25 500-519 52 52 52 105 105 105 T T T 26 520-539 52 52 52 105 105 105 T T T 27 540-559 48 48 48 101 101 101 F F F 28 560-579 51 51 51 104 104 104 F F F 29 580-599 48 48 48 101 101 101 F F F Max gen: 52, Max flush: 105 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 102 102 102 F F F 1 020-039 49 49 49 102 102 102 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 53 53 53 106 106 106 F F F 4 080-099 53 53 53 106 106 106 F F F 5 100-119 53 53 53 106 106 106 F F F 6 120-139 51 51 51 104 104 104 F F F 7 140-159 34 34 34 72 72 72 F F F 8 160-179 53 53 53 106 106 106 F F F 9 180-199 53 53 53 106 106 106 F F F 10 200-219 53 53 53 106 106 106 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 48 48 48 101 101 101 F F F 13 260-279 51 51 51 104 104 104 F F F 14 280-299 51 51 51 104 104 104 F F F 15 300-319 52 52 52 105 105 105 F F F 16 320-339 51 51 51 104 104 104 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 53 53 53 106 106 106 F F F 19 380-399 53 53 53 106 106 106 F F F 20 400-419 51 51 51 104 104 104 F F F 21 420-439 51 51 51 104 104 104 F F F 22 440-459 51 51 51 104 104 104 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 51 51 51 104 104 104 F F F 25 500-519 53 53 53 106 106 106 F F F 26 520-539 53 53 53 106 106 106 F F F 27 540-559 48 48 48 101 101 101 F F F 28 560-579 51 51 51 104 104 104 F F F 29 580-599 48 48 48 101 101 101 F F F Max gen: 53, Max flush: 106 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.789011385Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.832807068Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.833390183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.833435083Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.833450122Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.833464702Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408} {"msg":"Crucible c66f5add-d522-45b8-8936-dc50b2b2ef59 has session id: 94851a2d-fa2b-448f-a24c-31ce6a2eeced","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834006427Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Upstairs opts: Upstairs UUID: c66f5add-d522-45b8-8936-dc50b2b2ef59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834056107Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Crucible stats registered with UUID: c66f5add-d522-45b8-8936-dc50b2b2ef59","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834085217Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:24.834105546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834484873Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834510183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408} {"msg":"c66f5add-d522-45b8-8936-dc50b2b2ef59 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834571322Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834589312Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"0","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834618152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"0","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834632212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"1","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834646312Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"1","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834667781Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"2","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834695301Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"2","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834719381Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834747201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.8347729Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"ds_connection connected from Ok(127.0.0.1:51902)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.83479995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"ds_connection connected from Ok(127.0.0.1:34917)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834909499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"ds_connection connected from Ok(127.0.0.1:48739)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.834947329Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.835579033Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"2","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.835607153Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"0","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.836081128Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"1","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.836227927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"2","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.836325256Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"0","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.836799632Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"1","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.836870412Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.836907271Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[0]R flush_numbers[0..12]: [102, 102, 77, 106, 106, 106, 104, 72, 106, 106, 106, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837407467Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[0]R generation[0..12]: [49, 49, 37, 53, 53, 53, 51, 34, 53, 53, 53, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837432986Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837451656Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[1]R flush_numbers[0..12]: [102, 102, 77, 106, 106, 106, 104, 72, 106, 106, 106, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837465936Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[1]R generation[0..12]: [49, 49, 37, 53, 53, 53, 51, 34, 53, 53, 53, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837479506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837493566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[2]R flush_numbers[0..12]: [102, 102, 77, 106, 106, 106, 104, 72, 106, 106, 106, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837506956Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[2]R generation[0..12]: [49, 49, 37, 53, 53, 53, 51, 34, 53, 53, 53, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837519955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837533426Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837546025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837565295Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837577955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837590465Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"":"downstairs","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837603275Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837614885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"0","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837627854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"1","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837642665Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"client":"2","":"downstairs","session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837655854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837667844Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"c66f5add-d522-45b8-8936-dc50b2b2ef59 is now active with session: 94851a2d-fa2b-448f-a24c-31ce6a2eeced","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837680374Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837691834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408,"session_id":"94851a2d-fa2b-448f-a24c-31ce6a2eeced"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837703654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408} {"msg":"Activated sub_volume c66f5add-d522-45b8-8936-dc50b2b2ef59","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:24.837714614Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 567 len 20480 data: 5 5 4 4 5 03/30 Read block 206 len 36864 04/30 Write block 289 len 12288 data: 3 5 5 05/30 Read block 257 len 20480 06/30 Write block 529 len 20480 data: 6 4 2 4 5 07/30 Write block 260 len 36864 data: 3 3 4 4 7 6 5 7 7 08/30 Read block 394 len 12288 09/30 Read block 224 len 28672 10/30 Flush 11/30 Write block 133 len 20480 data: 3 4 4 4 5 12/30 Write block 535 len 24576 data: 7 6 5 7 7 7 13/30 Read block 398 len 24576 14/30 Read block 489 len 4096 15/30 Read block 495 len 40960 16/30 Read block 201 len 8192 17/30 Write block 524 len 28672 data: 4 5 6 7 6 7 5 18/30 Write block 189 len 24576 data: 3 3 3 4 4 4 19/30 Write block 420 len 28672 data: 8 6 6 5 4 5 6 20/30 Write block 500 len 20480 data: 4 4 3 3 3 21/30 Write block 429 len 16384 data: 6 6 6 5 22/30 Write block 268 len 24576 data: 8 7 7 7 6 6 23/30 Write block 323 len 36864 data: 5 5 6 4 5 5 6 6 6 24/30 Read block 109 len 24576 25/30 Read block 385 len 28672 26/30 Write block 173 len 12288 data: 4 4 5 27/30 Read block 253 len 16384 28/30 Read block 113 len 20480 29/30 Write block 309 len 16384 data: 6 5 4 4 30/30 Write block 147 len 24576 data: 2 3 3 3 4 6 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 5 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 19 18 18 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:24.856721212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 102 102 102 F F F 1 020-039 49 49 49 102 102 102 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 53 53 53 106 106 106 F F F 4 080-099 53 53 53 106 106 106 F F F 5 100-119 53 53 53 106 106 106 F F F 6 120-139 51 51 51 104 104 104 T T T 7 140-159 34 34 34 72 72 72 T T T 8 160-179 53 53 53 106 106 106 T T T 9 180-199 53 53 53 106 106 106 T T T 10 200-219 53 53 53 106 106 106 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 48 48 48 101 101 101 F F F 13 260-279 54 54 54 108 108 108 T T T 14 280-299 54 54 54 108 108 108 F F F 15 300-319 52 52 52 105 105 105 T T T 16 320-339 51 51 51 104 104 104 T T T 17 340-359 48 48 48 101 101 101 F F F 18 360-379 53 53 53 106 106 106 F F F 19 380-399 53 53 53 106 106 106 F F F 20 400-419 51 51 51 104 104 104 F F F 21 420-439 51 51 51 104 104 104 T T T 22 440-459 51 51 51 104 104 104 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 51 51 51 104 104 104 F F F 25 500-519 53 53 53 106 106 106 T T T 26 520-539 54 54 54 108 108 108 T T T 27 540-559 48 48 48 101 101 101 T T T 28 560-579 54 54 54 108 108 108 F F F 29 580-599 48 48 48 101 101 101 F F F Max gen: 54, Max flush: 108 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 102 102 102 F F F 1 020-039 49 49 49 102 102 102 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 53 53 53 106 106 106 F F F 4 080-099 53 53 53 106 106 106 F F F 5 100-119 53 53 53 106 106 106 F F F 6 120-139 55 55 55 109 109 109 F F F 7 140-159 55 55 55 109 109 109 F F F 8 160-179 55 55 55 109 109 109 F F F 9 180-199 55 55 55 109 109 109 F F F 10 200-219 53 53 53 106 106 106 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 48 48 48 101 101 101 F F F 13 260-279 55 55 55 109 109 109 F F F 14 280-299 54 54 54 108 108 108 F F F 15 300-319 55 55 55 109 109 109 F F F 16 320-339 55 55 55 109 109 109 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 53 53 53 106 106 106 F F F 19 380-399 53 53 53 106 106 106 F F F 20 400-419 51 51 51 104 104 104 F F F 21 420-439 55 55 55 109 109 109 F F F 22 440-459 51 51 51 104 104 104 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 51 51 51 104 104 104 F F F 25 500-519 55 55 55 109 109 109 F F F 26 520-539 55 55 55 109 109 109 F F F 27 540-559 55 55 55 109 109 109 F F F 28 560-579 54 54 54 108 108 108 F F F 29 580-599 48 48 48 101 101 101 F F F Max gen: 55, Max flush: 109 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.126310811Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.171120636Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.17178025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.171821879Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.171836839Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.171852099Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416} {"msg":"Crucible fbddc524-8e66-43ae-8160-7432dbc95454 has session id: f01034fb-17b4-4b0e-89b5-22f83347946a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.172547083Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Upstairs opts: Upstairs UUID: fbddc524-8e66-43ae-8160-7432dbc95454, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.172589202Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Crucible stats registered with UUID: fbddc524-8e66-43ae-8160-7432dbc95454","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.172607712Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:29.172621372Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173049008Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173077118Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416} {"msg":"fbddc524-8e66-43ae-8160-7432dbc95454 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173145967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173164267Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"0","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173192777Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"0","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173206437Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"1","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173220346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"1","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173233286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"2","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173246936Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"2","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173260306Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173281336Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173295176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53879)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173311156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45230)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173386075Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44177)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.173418285Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.174054949Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"0","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.174090059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"1","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.174301457Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"2","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.174766532Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"0","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.174796552Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"2","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.174812862Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"1","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175401887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175424346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[0]R flush_numbers[0..12]: [102, 102, 77, 106, 106, 106, 109, 109, 109, 109, 106, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175449086Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[0]R generation[0..12]: [49, 49, 37, 53, 53, 53, 55, 55, 55, 55, 53, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175463856Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175478036Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[1]R flush_numbers[0..12]: [102, 102, 77, 106, 106, 106, 109, 109, 109, 109, 106, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175491606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[1]R generation[0..12]: [49, 49, 37, 53, 53, 53, 55, 55, 55, 55, 53, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175505116Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175518916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[2]R flush_numbers[0..12]: [102, 102, 77, 106, 106, 106, 109, 109, 109, 109, 106, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175532566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[2]R generation[0..12]: [49, 49, 37, 53, 53, 53, 55, 55, 55, 55, 53, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175550105Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175564795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175577505Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175590215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175602985Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175615725Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"":"downstairs","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175628705Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175640455Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"0","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175653415Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"1","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:43:29.175666204Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"client":"2","":"downstairs","session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175682564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175694524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"fbddc524-8e66-43ae-8160-7432dbc95454 is now active with session: f01034fb-17b4-4b0e-89b5-22f83347946a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175706964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175718694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416,"session_id":"f01034fb-17b4-4b0e-89b5-22f83347946a"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175730564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416} {"msg":"Activated sub_volume fbddc524-8e66-43ae-8160-7432dbc95454","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:29.175741544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 480 len 40960 data: 4 4 3 4 5 5 6 5 4 5 02/30 Write block 252 len 36864 data: 5 5 5 6 6 7 6 3 4 03/30 Write block 323 len 36864 data: 6 6 7 5 6 6 7 7 7 04/30 Read block 133 len 36864 05/30 Write block 529 len 28672 data: 8 6 3 5 6 6 8 06/30 Flush 07/30 Write block 203 len 8192 data: 6 6 08/30 Read block 478 len 40960 09/30 Write block 204 len 24576 data: 7 5 4 7 7 7 10/30 Read block 72 len 24576 11/30 Write block 72 len 8192 data: 4 6 12/30 Write block 98 len 24576 data: 6 6 6 7 7 7 13/30 Write block 28 len 4096 data: 6 14/30 Write block 9 len 40960 data: 6 7 8 6 5 4 5 5 5 5 15/30 Read block 365 len 4096 16/30 Write block 326 len 40960 data: 6 7 7 8 8 8 4 4 6 7 17/30 Write block 517 len 20480 data: 9 7 8 7 8 18/30 Read block 509 len 28672 19/30 Flush 20/30 Write block 590 len 28672 data: 3 3 4 4 4 4 4 21/30 Read block 213 len 8192 22/30 Flush 23/30 Write block 509 len 20480 data: 5 5 5 5 6 24/30 Read block 285 len 40960 25/30 Read block 220 len 8192 26/30 Read block 439 len 16384 27/30 Write block 88 len 4096 data: 5 28/30 Read block 520 len 40960 29/30 Read block 527 len 8192 30/30 Read block 233 len 12288 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:29.19600964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 111 111 111 F F F 1 020-039 56 56 56 111 111 111 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 56 56 56 111 111 111 F F F 4 080-099 56 56 56 111 111 111 T T T 5 100-119 56 56 56 111 111 111 F F F 6 120-139 55 55 55 109 109 109 F F F 7 140-159 55 55 55 109 109 109 F F F 8 160-179 55 55 55 109 109 109 F F F 9 180-199 55 55 55 109 109 109 F F F 10 200-219 56 56 56 111 111 111 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 56 56 56 110 110 110 F F F 13 260-279 56 56 56 110 110 110 F F F 14 280-299 54 54 54 108 108 108 F F F 15 300-319 55 55 55 109 109 109 F F F 16 320-339 56 56 56 111 111 111 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 53 53 53 106 106 106 F F F 19 380-399 53 53 53 106 106 106 F F F 20 400-419 51 51 51 104 104 104 F F F 21 420-439 55 55 55 109 109 109 F F F 22 440-459 51 51 51 104 104 104 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 56 56 56 111 111 111 T T T 26 520-539 56 56 56 111 111 111 F F F 27 540-559 55 55 55 109 109 109 F F F 28 560-579 54 54 54 108 108 108 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 56, Max flush: 112 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 111 111 111 F F F 1 020-039 56 56 56 111 111 111 F F F 2 040-059 37 37 37 77 77 77 F F F 3 060-079 56 56 56 111 111 111 F F F 4 080-099 57 57 57 113 113 113 F F F 5 100-119 56 56 56 111 111 111 F F F 6 120-139 55 55 55 109 109 109 F F F 7 140-159 55 55 55 109 109 109 F F F 8 160-179 55 55 55 109 109 109 F F F 9 180-199 55 55 55 109 109 109 F F F 10 200-219 56 56 56 111 111 111 F F F 11 220-239 48 48 48 101 101 101 F F F 12 240-259 56 56 56 110 110 110 F F F 13 260-279 56 56 56 110 110 110 F F F 14 280-299 54 54 54 108 108 108 F F F 15 300-319 55 55 55 109 109 109 F F F 16 320-339 56 56 56 111 111 111 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 53 53 53 106 106 106 F F F 19 380-399 53 53 53 106 106 106 F F F 20 400-419 51 51 51 104 104 104 F F F 21 420-439 55 55 55 109 109 109 F F F 22 440-459 51 51 51 104 104 104 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 57 57 57 113 113 113 F F F 26 520-539 56 56 56 111 111 111 F F F 27 540-559 55 55 55 109 109 109 F F F 28 560-579 54 54 54 108 108 108 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 57, Max flush: 113 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.415317074Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.459066348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.459677343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.459727932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.459745472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.459763252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424} {"msg":"Crucible 2e4290c4-3fa5-4e62-a3ee-390f7cb97bec has session id: 61b977a4-24f8-489e-8113-2bcc702f24de","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460310607Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Upstairs opts: Upstairs UUID: 2e4290c4-3fa5-4e62-a3ee-390f7cb97bec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460343087Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Crucible stats registered with UUID: 2e4290c4-3fa5-4e62-a3ee-390f7cb97bec","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460358556Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:33.460371167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460811243Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460840502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424} {"msg":"2e4290c4-3fa5-4e62-a3ee-390f7cb97bec active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460868212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460881132Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"0","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460911331Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"0","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460925122Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"1","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460939191Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"1","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460952061Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"2","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460969591Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"2","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.460993921Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.46106314Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.46109788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"ds_connection connected from Ok(127.0.0.1:46901)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.461208299Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"ds_connection connected from Ok(127.0.0.1:54114)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.461282458Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"ds_connection connected from Ok(127.0.0.1:48749)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.461320648Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.461951392Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"2","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.461985472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"0","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.462445468Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"1","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.462624386Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"0","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.462663706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"2","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.462999543Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"1","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463236821Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.46332566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[0]R flush_numbers[0..12]: [111, 111, 77, 111, 113, 111, 109, 109, 109, 109, 111, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463577447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[0]R generation[0..12]: [56, 56, 37, 56, 57, 56, 55, 55, 55, 55, 56, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463605317Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463621947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[1]R flush_numbers[0..12]: [111, 111, 77, 111, 113, 111, 109, 109, 109, 109, 111, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463636267Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[1]R generation[0..12]: [56, 56, 37, 56, 57, 56, 55, 55, 55, 55, 56, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463649657Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463663317Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[2]R flush_numbers[0..12]: [111, 111, 77, 111, 113, 111, 109, 109, 109, 109, 111, 101]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463676706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[2]R generation[0..12]: [56, 56, 37, 56, 57, 56, 55, 55, 55, 55, 56, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463689846Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463703486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463716196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463729276Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463747926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463760756Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"":"downstairs","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463773716Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463785385Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"0","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463798256Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"1","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463813325Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"client":"2","":"downstairs","session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463826995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463838905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"2e4290c4-3fa5-4e62-a3ee-390f7cb97bec is now active with session: 61b977a4-24f8-489e-8113-2bcc702f24de","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463851425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463863045Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424,"session_id":"61b977a4-24f8-489e-8113-2bcc702f24de"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463874855Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424} {"msg":"Activated sub_volume 2e4290c4-3fa5-4e62-a3ee-390f7cb97bec","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:33.463885855Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 6 len 12288 data: 5 8 7 03/30 Read block 214 len 20480 04/30 Read block 389 len 20480 05/30 Write block 51 len 24576 data: 6 8 5 7 7 7 06/30 Write block 155 len 28672 data: 7 7 4 4 4 4 5 07/30 Write block 397 len 28672 data: 9 6 6 7 8 8 8 08/30 Write block 120 len 32768 data: 7 8 8 9 7 5 5 5 09/30 Flush 10/30 Write block 579 len 4096 data: 8 11/30 Write block 551 len 36864 data: 6 3 4 3 4 3 4 3 4 12/30 Read block 438 len 16384 13/30 Flush 14/30 Write block 25 len 20480 data: 2 4 5 7 6 15/30 Write block 443 len 32768 data: 4 7 6 6 7 6 5 4 16/30 Write block 209 len 8192 data: 8 6 17/30 Read block 124 len 32768 18/30 Write block 503 len 8192 data: 4 4 19/30 Write block 233 len 32768 data: 7 9 10 7 6 7 6 9 20/30 Read block 598 len 8192 21/30 Write block 533 len 4096 data: 7 22/30 Write block 265 len 12288 data: 7 6 8 23/30 Write block 526 len 4096 data: 7 24/30 Write block 194 len 24576 data: 5 3 5 5 6 6 25/30 Write block 61 len 36864 data: 8 9 8 8 7 6 6 7 6 26/30 Read block 102 len 4096 27/30 Write block 424 len 4096 data: 5 28/30 Read block 99 len 20480 29/30 Write block 416 len 12288 data: 8 7 7 30/30 Read block 451 len 16384 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 5 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 0 2 Done 17 15 17 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:33.481635624Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 F F F 1 020-039 56 56 56 111 111 111 T T T 2 040-059 58 58 58 115 115 115 F F F 3 060-079 56 56 56 111 111 111 T T T 4 080-099 57 57 57 113 113 113 F F F 5 100-119 56 56 56 111 111 111 F F F 6 120-139 58 58 58 115 115 115 F F F 7 140-159 58 58 58 115 115 115 F F F 8 160-179 58 58 58 115 115 115 F F F 9 180-199 55 55 55 109 109 109 T T T 10 200-219 56 56 56 111 111 111 T T T 11 220-239 48 48 48 101 101 101 T T T 12 240-259 56 56 56 110 110 110 T T T 13 260-279 56 56 56 110 110 110 T T T 14 280-299 54 54 54 108 108 108 F F F 15 300-319 55 55 55 109 109 109 F F F 16 320-339 56 56 56 111 111 111 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 53 53 53 106 106 106 F F F 19 380-399 58 58 58 115 115 115 F F F 20 400-419 58 58 58 115 115 115 T T T 21 420-439 55 55 55 109 109 109 T T T 22 440-459 51 51 51 104 104 104 T T T 23 460-479 50 50 50 103 103 103 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 57 57 57 113 113 113 T T T 26 520-539 56 56 56 111 111 111 T T T 27 540-559 58 58 58 116 116 116 F F F 28 560-579 58 58 58 116 116 116 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 58, Max flush: 116 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 F F F 1 020-039 59 59 59 117 117 117 F F F 2 040-059 58 58 58 115 115 115 F F F 3 060-079 59 59 59 117 117 117 F F F 4 080-099 57 57 57 113 113 113 F F F 5 100-119 56 56 56 111 111 111 F F F 6 120-139 58 58 58 115 115 115 F F F 7 140-159 58 58 58 115 115 115 F F F 8 160-179 58 58 58 115 115 115 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 59 59 59 117 117 117 F F F 11 220-239 59 59 59 117 117 117 F F F 12 240-259 59 59 59 117 117 117 F F F 13 260-279 59 59 59 117 117 117 F F F 14 280-299 54 54 54 108 108 108 F F F 15 300-319 55 55 55 109 109 109 F F F 16 320-339 56 56 56 111 111 111 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 53 53 53 106 106 106 F F F 19 380-399 58 58 58 115 115 115 F F F 20 400-419 59 59 59 117 117 117 F F F 21 420-439 59 59 59 117 117 117 F F F 22 440-459 59 59 59 117 117 117 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 59 59 59 117 117 117 F F F 26 520-539 59 59 59 117 117 117 F F F 27 540-559 58 58 58 116 116 116 F F F 28 560-579 58 58 58 116 116 116 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 59, Max flush: 117 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.751432111Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.795013666Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.795606671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.79566475Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.79568026Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.79569503Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432} {"msg":"Crucible b27eda45-0d30-4aef-ba6f-0f2b382ac9e8 has session id: df613d04-58b1-46e7-9cf6-03634e837b47","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.796390734Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Upstairs opts: Upstairs UUID: b27eda45-0d30-4aef-ba6f-0f2b382ac9e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.796425883Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Crucible stats registered with UUID: b27eda45-0d30-4aef-ba6f-0f2b382ac9e8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.796442114Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:37.796455263Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.796905599Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.796928909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432} {"msg":"b27eda45-0d30-4aef-ba6f-0f2b382ac9e8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797003158Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797022558Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"0","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797052198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"0","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797070468Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"1","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797095688Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"1","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797121257Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"2","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797148837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"2","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797175017Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797202447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797229696Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"ds_connection connected from Ok(127.0.0.1:36872)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797274186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"ds_connection connected from Ok(127.0.0.1:52472)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797301726Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"ds_connection connected from Ok(127.0.0.1:56573)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797330755Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.7979545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"0","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.797993359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"1","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.798496885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"2","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.798592194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"0","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.798620914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"1","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.79908175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"2","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799291848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799310868Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[0]R flush_numbers[0..12]: [115, 117, 115, 117, 113, 111, 115, 115, 115, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799710124Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[0]R generation[0..12]: [58, 59, 58, 59, 57, 56, 58, 58, 58, 59, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799731594Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799746684Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[1]R flush_numbers[0..12]: [115, 117, 115, 117, 113, 111, 115, 115, 115, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799760473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[1]R generation[0..12]: [58, 59, 58, 59, 57, 56, 58, 58, 58, 59, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799773803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799787333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[2]R flush_numbers[0..12]: [115, 117, 115, 117, 113, 111, 115, 115, 115, 117, 117, 117]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799800613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[2]R generation[0..12]: [58, 59, 58, 59, 57, 56, 58, 58, 58, 59, 59, 59]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799817523Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799832133Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799844883Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799857703Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799870262Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799882912Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"":"downstairs","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799895642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799907232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"0","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799920122Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"1","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:43:37.799932812Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"client":"2","":"downstairs","session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799952132Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799964251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"b27eda45-0d30-4aef-ba6f-0f2b382ac9e8 is now active with session: df613d04-58b1-46e7-9cf6-03634e837b47","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799976982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.799989711Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432,"session_id":"df613d04-58b1-46e7-9cf6-03634e837b47"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.800001741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432} {"msg":"Activated sub_volume b27eda45-0d30-4aef-ba6f-0f2b382ac9e8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:37.800012941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 101 len 28672 data: 8 8 8 4 6 6 5 03/30 Read block 218 len 32768 04/30 Read block 457 len 36864 05/30 Read block 100 len 12288 06/30 Write block 26 len 12288 data: 5 6 8 07/30 Read block 258 len 16384 08/30 Write block 32 len 24576 data: 7 5 5 6 7 8 09/30 Write block 221 len 20480 data: 10 9 8 8 6 10/30 Write block 337 len 4096 data: 7 11/30 Write block 548 len 40960 data: 8 9 7 7 4 5 4 5 4 5 12/30 Read block 249 len 32768 13/30 Write block 175 len 4096 data: 6 14/30 Read block 117 len 28672 15/30 Read block 114 len 40960 16/30 Flush 17/30 Write block 221 len 40960 data: 11 10 9 9 7 5 6 6 7 6 18/30 Read block 177 len 16384 19/30 Read block 55 len 28672 20/30 Write block 362 len 4096 data: 4 21/30 Read block 209 len 4096 22/30 Write block 60 len 8192 data: 6 9 23/30 Read block 398 len 8192 24/30 Write block 309 len 32768 data: 7 6 5 5 4 4 4 6 25/30 Read block 337 len 8192 26/30 Read block 12 len 24576 27/30 Write block 65 len 32768 data: 8 7 7 8 7 4 5 5 28/30 Read block 276 len 20480 29/30 Write block 56 len 28672 data: 8 5 3 3 7 10 10 30/30 Write block 524 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:37.820175319Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 F F F 1 020-039 60 60 60 119 119 119 F F F 2 040-059 58 58 58 115 115 115 T T T 3 060-079 59 59 59 117 117 117 T T T 4 080-099 57 57 57 113 113 113 F F F 5 100-119 60 60 60 119 119 119 F F F 6 120-139 58 58 58 115 115 115 F F F 7 140-159 58 58 58 115 115 115 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 59 59 59 117 117 117 F F F 11 220-239 60 60 60 119 119 119 T T T 12 240-259 59 59 59 117 117 117 F F F 13 260-279 59 59 59 117 117 117 F F F 14 280-299 54 54 54 108 108 108 F F F 15 300-319 55 55 55 109 109 109 T T T 16 320-339 60 60 60 119 119 119 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 53 53 53 106 106 106 T T T 19 380-399 58 58 58 115 115 115 F F F 20 400-419 59 59 59 117 117 117 F F F 21 420-439 59 59 59 117 117 117 F F F 22 440-459 59 59 59 117 117 117 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 59 59 59 117 117 117 F F F 26 520-539 59 59 59 117 117 117 T T T 27 540-559 60 60 60 119 119 119 F F F 28 560-579 58 58 58 116 116 116 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 60, Max flush: 119 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 F F F 1 020-039 60 60 60 119 119 119 F F F 2 040-059 61 61 61 120 120 120 F F F 3 060-079 61 61 61 120 120 120 F F F 4 080-099 57 57 57 113 113 113 F F F 5 100-119 60 60 60 119 119 119 F F F 6 120-139 58 58 58 115 115 115 F F F 7 140-159 58 58 58 115 115 115 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 59 59 59 117 117 117 F F F 11 220-239 61 61 61 120 120 120 F F F 12 240-259 59 59 59 117 117 117 F F F 13 260-279 59 59 59 117 117 117 F F F 14 280-299 54 54 54 108 108 108 F F F 15 300-319 61 61 61 120 120 120 F F F 16 320-339 60 60 60 119 119 119 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 61 61 61 120 120 120 F F F 19 380-399 58 58 58 115 115 115 F F F 20 400-419 59 59 59 117 117 117 F F F 21 420-439 59 59 59 117 117 117 F F F 22 440-459 59 59 59 117 117 117 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 59 59 59 117 117 117 F F F 26 520-539 61 61 61 120 120 120 F F F 27 540-559 60 60 60 119 119 119 F F F 28 560-579 58 58 58 116 116 116 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 61, Max flush: 120 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.064613545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.109650888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.110284362Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.110327061Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.110341551Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.110360801Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440} {"msg":"Crucible c0b2c9d8-ce55-4bf5-ac7f-963f24466a72 has session id: c7ef061e-041a-4452-a75f-89010fe36774","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.110855517Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Upstairs opts: Upstairs UUID: c0b2c9d8-ce55-4bf5-ac7f-963f24466a72, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.110897176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Crucible stats registered with UUID: c0b2c9d8-ce55-4bf5-ac7f-963f24466a72","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.110920686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:42.110941156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111289413Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111313662Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440} {"msg":"c0b2c9d8-ce55-4bf5-ac7f-963f24466a72 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111372672Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111416652Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"0","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111447201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"0","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111461741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"1","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111476511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"1","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111490001Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"2","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111503921Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"2","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.11160284Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.11162859Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111657709Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"ds_connection connected from Ok(127.0.0.1:63362)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111683259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"ds_connection connected from Ok(127.0.0.1:36042)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111709409Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"ds_connection connected from Ok(127.0.0.1:64764)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.111788488Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.112371023Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"0","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.112401303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"2","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.112496982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"1","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.112982147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"0","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113027617Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"2","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113182596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"1","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113619392Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113652731Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[0]R flush_numbers[0..12]: [115, 119, 120, 120, 113, 119, 115, 115, 119, 117, 117, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.11381369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[0]R generation[0..12]: [58, 60, 61, 61, 57, 60, 58, 58, 60, 59, 59, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.1138399Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113855219Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[1]R flush_numbers[0..12]: [115, 119, 120, 120, 113, 119, 115, 115, 119, 117, 117, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113869619Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[1]R generation[0..12]: [58, 60, 61, 61, 57, 60, 58, 58, 60, 59, 59, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113882999Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113896869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[2]R flush_numbers[0..12]: [115, 119, 120, 120, 113, 119, 115, 115, 119, 117, 117, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113910379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[2]R generation[0..12]: [58, 60, 61, 61, 57, 60, 58, 58, 60, 59, 59, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113923749Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113937289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113950039Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113962778Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113975568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.113988258Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"":"downstairs","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114001018Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114012688Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"0","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114027848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"1","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114040838Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"client":"2","":"downstairs","session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114053638Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114065468Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"c0b2c9d8-ce55-4bf5-ac7f-963f24466a72 is now active with session: c7ef061e-041a-4452-a75f-89010fe36774","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114077917Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114089397Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440,"session_id":"Wait for a query_work_queue command to finish before sending IO c7ef061e-041a-4452-a75f-89010fe36774"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114109057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440} {"msg":"Activated sub_volume c0b2c9d8-ce55-4bf5-ac7f-963f24466a72","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:42.114120127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 137 len 16384 data: 6 4 3 4 02/30 Read block 150 len 20480 03/30 Read block 374 len 8192 04/30 Read block 34 len 32768 05/30 Read block 125 len 4096 06/30 Write block 65 len 8192 data: 9 8 07/30 Read block 166 len 24576 08/30 Read block 494 len 4096 09/30 Read block 393 len 16384 10/30 Read block 473 len 24576 11/30 Write block 60 len 36864 data: 8 11 11 9 9 10 9 8 9 12/30 Read block 598 len 8192 13/30 Read block 328 len 16384 14/30 Read block 556 len 12288 15/30 Read block 460 len 8192 16/30 Read block 347 len 28672 17/30 Read block 133 len 20480 18/30 Write block 569 len 28672 data: 5 5 6 5 3 4 4 19/30 Write block 41 len 20480 data: 6 5 6 6 5 20/30 Write block 489 len 28672 data: 6 5 5 5 5 5 4 21/30 Write block 377 len 40960 data: 8 8 7 6 4 5 5 4 5 4 22/30 Read block 514 len 16384 23/30 Read block 211 len 16384 24/30 Flush 25/30 Write block 292 len 28672 data: 4 4 5 5 5 4 4 26/30 Read block 195 len 4096 27/30 Write block 383 len 32768 data: 6 5 6 5 6 7 7 8 28/30 Read block 312 len 20480 29/30 Read block 138 len 12288 30/30 Write block 206 len 24576 data: 5 8 8 9 7 6 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:42.134735261Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 F F F 1 020-039 60 60 60 119 119 119 F F F 2 040-059 62 62 62 121 121 121 F F F 3 060-079 62 62 62 121 121 121 F F F 4 080-099 57 57 57 113 113 113 F F F 5 100-119 60 60 60 119 119 119 F F F 6 120-139 62 62 62 121 121 121 F F F 7 140-159 62 62 62 121 121 121 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 59 59 59 117 117 117 T T T 11 220-239 61 61 61 120 120 120 F F F 12 240-259 59 59 59 117 117 117 F F F 13 260-279 59 59 59 117 117 117 F F F 14 280-299 54 54 54 108 108 108 T T T 15 300-319 61 61 61 120 120 120 F F F 16 320-339 60 60 60 119 119 119 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 62 62 62 121 121 121 T T T 20 400-419 59 59 59 117 117 117 F F F 21 420-439 59 59 59 117 117 117 F F F 22 440-459 59 59 59 117 117 117 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 62 62 62 121 121 121 F F F 25 500-519 59 59 59 117 117 117 F F F 26 520-539 61 61 61 120 120 120 F F F 27 540-559 60 60 60 119 119 119 F F F 28 560-579 62 62 62 121 121 121 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 62, Max flush: 121 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 F F F 1 020-039 60 60 60 119 119 119 F F F 2 040-059 62 62 62 121 121 121 F F F 3 060-079 62 62 62 121 121 121 F F F 4 080-099 57 57 57 113 113 113 F F F 5 100-119 60 60 60 119 119 119 F F F 6 120-139 62 62 62 121 121 121 F F F 7 140-159 62 62 62 121 121 121 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 63 63 63 122 122 122 F F F 11 220-239 61 61 61 120 120 120 F F F 12 240-259 59 59 59 117 117 117 F F F 13 260-279 59 59 59 117 117 117 F F F 14 280-299 63 63 63 122 122 122 F F F 15 300-319 61 61 61 120 120 120 F F F 16 320-339 60 60 60 119 119 119 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 63 63 63 122 122 122 F F F 20 400-419 59 59 59 117 117 117 F F F 21 420-439 59 59 59 117 117 117 F F F 22 440-459 59 59 59 117 117 117 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 62 62 62 121 121 121 F F F 25 500-519 59 59 59 117 117 117 F F F 26 520-539 61 61 61 120 120 120 F F F 27 540-559 60 60 60 119 119 119 F F F 28 560-579 62 62 62 121 121 121 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 63, Max flush: 122 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.360903932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.405072743Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.405747877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.405793126Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.405807676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.405821916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448} {"msg":"Crucible 43534d5b-7ff7-4ec9-8deb-405885aca2d9 has session id: 39190d72-94e5-4f74-b153-15d93c87527d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406310911Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Upstairs opts: Upstairs UUID: 43534d5b-7ff7-4ec9-8deb-405885aca2d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406367251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Crucible stats registered with UUID: 43534d5b-7ff7-4ec9-8deb-405885aca2d9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406392771Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:46.40641348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406823807Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406850947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448} {"msg":"43534d5b-7ff7-4ec9-8deb-405885aca2d9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406893736Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406909946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"0","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406944196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"0","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406958735Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"1","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406972935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"1","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.406985945Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"2","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407011875Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"2","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407034345Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407058525Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407081834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46108)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407106274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60470)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407354662Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44556)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407390871Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407861417Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"0","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.407892637Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"2","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.408086115Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"1","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.40865564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"0","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.40869664Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"2","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.40873061Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"1","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409261485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409374254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[0]R flush_numbers[0..12]: [115, 119, 121, 121, 113, 119, 121, 121, 119, 117, 122, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409436343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[0]R generation[0..12]: [58, 60, 62, 62, 57, 60, 62, 62, 60, 59, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409464233Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409479843Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[1]R flush_numbers[0..12]: [115, 119, 121, 121, 113, 119, 121, 121, 119, 117, 122, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409494592Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[1]R generation[0..12]: [58, 60, 62, 62, 57, 60, 62, 62, 60, 59, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409508692Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409522742Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[2]R flush_numbers[0..12]: [115, 119, 121, 121, 113, 119, 121, 121, 119, 117, 122, 120]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409536642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[2]R generation[0..12]: [58, 60, 62, 62, 57, 60, 62, 62, 60, 59, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409550122Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409563912Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409580342Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409593792Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409606602Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409619331Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"":"downstairs","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409632531Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409644401Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"0","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409663521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"1","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409676561Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"client":"2","":"downstairs","session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409689631Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.409701601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"43534d5b-7ff7-4ec9-8deb-405885aca2d9 is now active with session: 39190d72-94e5-4f74-b153-15d93c87527d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.40971427Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.40972607Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448,"session_id":"39190d72-94e5-4f74-b153-15d93c87527d"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.40973799Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448} {"msg":"Activated sub_volume 43534d5b-7ff7-4ec9-8deb-405885aca2d9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:46.40974894Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 392 len 36864 data: 8 7 7 7 9 10 7 7 8 02/30 Read block 12 len 36864 03/30 Write block 42 len 24576 data: 6 7 7 6 4 5 04/30 Read block 26 len 32768 05/30 Read block 504 len 16384 06/30 Read block 337 len 28672 07/30 Read block 43 len 24576 08/30 Read block 89 len 28672 09/30 Write block 221 len 16384 data: 12 11 10 10 10/30 Write block 283 len 40960 data: 8 7 4 4 4 3 4 6 6 5 11/30 Read block 223 len 4096 12/30 Flush 13/30 Write block 106 len 32768 data: 7 6 6 5 7 7 5 6 14/30 Write block 69 len 8192 data: 8 5 15/30 Write block 504 len 16384 data: 5 3 4 6 16/30 Write block 448 len 8192 data: 7 6 17/30 Read block 516 len 24576 18/30 Read block 139 len 4096 19/30 Read block 442 len 12288 20/30 Write block 454 len 24576 data: 5 5 5 5 5 3 21/30 Write block 148 len 16384 data: 4 4 4 5 22/30 Read block 34 len 12288 23/30 Read block 429 len 16384 24/30 Write block 325 len 32768 data: 8 7 8 8 9 9 9 5 25/30 Read block 270 len 40960 26/30 Read block 16 len 32768 27/30 Read block 480 len 12288 28/30 Read block 385 len 36864 29/30 Read block 289 len 16384 30/30 Read block 597 len 12288 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 8 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 17 18 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:46.430423583Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 F F F 1 020-039 60 60 60 119 119 119 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 62 62 62 121 121 121 T T T 4 080-099 57 57 57 113 113 113 F F F 5 100-119 60 60 60 119 119 119 T T T 6 120-139 62 62 62 121 121 121 F F F 7 140-159 62 62 62 121 121 121 T T T 8 160-179 60 60 60 119 119 119 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 63 63 63 122 122 122 F F F 11 220-239 64 64 64 123 123 123 F F F 12 240-259 59 59 59 117 117 117 F F F 13 260-279 59 59 59 117 117 117 F F F 14 280-299 64 64 64 123 123 123 F F F 15 300-319 61 61 61 120 120 120 F F F 16 320-339 60 60 60 119 119 119 T T T 17 340-359 48 48 48 101 101 101 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 64 64 64 123 123 123 F F F 20 400-419 64 64 64 123 123 123 F F F 21 420-439 59 59 59 117 117 117 F F F 22 440-459 59 59 59 117 117 117 T T T 23 460-479 50 50 50 103 103 103 F F F 24 480-499 62 62 62 121 121 121 F F F 25 500-519 59 59 59 117 117 117 T T T 26 520-539 61 61 61 120 120 120 F F F 27 540-559 60 60 60 119 119 119 F F F 28 560-579 62 62 62 121 121 121 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 64, Max flush: 123 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 F F F 1 020-039 60 60 60 119 119 119 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 65 65 65 124 124 124 F F F 4 080-099 57 57 57 113 113 113 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 62 62 62 121 121 121 F F F 7 140-159 65 65 65 124 124 124 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 63 63 63 122 122 122 F F F 11 220-239 64 64 64 123 123 123 F F F 12 240-259 59 59 59 117 117 117 F F F 13 260-279 59 59 59 117 117 117 F F F 14 280-299 64 64 64 123 123 123 F F F 15 300-319 61 61 61 120 120 120 F F F 16 320-339 65 65 65 124 124 124 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 64 64 64 123 123 123 F F F 20 400-419 64 64 64 123 123 123 F F F 21 420-439 59 59 59 117 117 117 F F F 22 440-459 65 65 65 124 124 124 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 62 62 62 121 121 121 F F F 25 500-519 65 65 65 124 124 124 F F F 26 520-539 61 61 61 120 120 120 F F F 27 540-559 60 60 60 119 119 119 F F F 28 560-579 62 62 62 121 121 121 F F F 29 580-599 56 56 56 112 112 112 F F F Max gen: 65, Max flush: 124 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.67702236Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.720784424Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.721361969Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.721406299Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.721420798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.721434828Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456} {"msg":"Crucible 2fcd137c-3abb-4a29-aaa7-1ba201a0d95f has session id: d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722069663Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Upstairs opts: Upstairs UUID: 2fcd137c-3abb-4a29-aaa7-1ba201a0d95f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722109202Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Crucible stats registered with UUID: 2fcd137c-3abb-4a29-aaa7-1ba201a0d95f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722131372Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:50.722150302Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722629318Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722655857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456} {"msg":"2fcd137c-3abb-4a29-aaa7-1ba201a0d95f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722719657Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722737337Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"0","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722767326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"0","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722783086Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"1","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722807336Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"1","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722830476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"2","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722873615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"2","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722898315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722921935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722949865Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:40448)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.722974635Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35841)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.723002264Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:48781)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.723029184Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.723658538Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"0","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.723699758Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"1","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.724312112Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"0","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.724345702Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"1","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.724957977Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.724987186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.725485602Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"2","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.725978727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"2","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[0]R flush_numbers[0..12]: [115, 119, 123, 124, 113, 124, 121, 124, 119, 117, 122, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726482523Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[0]R generation[0..12]: [58, 60, 64, 65, 57, 65, 62, 65, 60, 59, 63, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726503512Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726522442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[1]R flush_numbers[0..12]: [115, 119, 123, 124, 113, 124, 121, 124, 119, 117, 122, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726537902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[1]R generation[0..12]: [58, 60, 64, 65, 57, 65, 62, 65, 60, 59, 63, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726551442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726566162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[2]R flush_numbers[0..12]: [115, 119, 123, 124, 113, 124, 121, 124, 119, 117, 122, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726579662Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[2]R generation[0..12]: [58, 60, 64, 65, 57, 65, 62, 65, 60, 59, 63, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726593432Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726607091Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726620462Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726633991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Next flush: 125","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:43:50.726646521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726665811Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"":"downstairs","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726678501Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726690061Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"0","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726702881Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"1","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726715741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"client":"2","":"downstairs","session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.726730941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.72674382Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"2fcd137c-3abb-4a29-aaa7-1ba201a0d95f is now active with session: d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.7267565Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.72676903Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456,"session_id":"d468dbb1-c00e-4c11-9d7e-ef5ccedc88fc"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.7267812Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456} {"msg":"Activated sub_volume 2fcd137c-3abb-4a29-aaa7-1ba201a0d95f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:43:50.72679281Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 424 len 28672 data: 6 6 7 5 7 7 7 02/30 Write block 549 len 36864 data: 10 8 8 5 6 5 6 5 6 03/30 Write block 396 len 32768 data: 10 11 8 8 9 9 9 9 04/30 Write block 86 len 32768 data: 5 4 6 7 7 9 9 8 05/30 Read block 196 len 24576 06/30 Write block 592 len 8192 data: 5 5 07/30 Write block 503 len 24576 data: 5 6 4 5 7 5 08/30 Read block 475 len 28672 09/30 Write block 61 len 40960 data: 12 12 10 10 11 10 9 10 9 6 10/30 Flush 11/30 Write block 250 len 40960 data: 4 6 6 6 6 7 7 8 7 4 12/30 Read block 411 len 12288 13/30 Read block 206 len 8192 14/30 Read block 557 len 32768 15/30 Read block 340 len 20480 16/30 Write block 17 len 16384 data: 6 6 6 5 17/30 Write block 248 len 28672 data: 4 4 5 7 7 7 7 18/30 Write block 5 len 32768 data: 4 6 9 8 7 8 9 7 19/30 Write block 146 len 16384 data: 3 3 5 5 20/30 Write block 137 len 28672 data: 7 5 4 5 4 4 3 21/30 Write block 488 len 8192 data: 5 7 22/30 Write block 538 len 32768 data: 8 8 8 7 6 7 6 7 23/30 Write block 587 len 36864 data: 6 5 3 4 4 6 6 5 5 24/30 Read block 286 len 40960 25/30 Write block 89 len 12288 data: 8 8 10 26/30 Write block 206 len 40960 data: 6 9 9 10 8 7 6 6 8 5 27/30 Read block 532 len 16384 28/30 Read block 516 len 12288 29/30 Write block 82 len 28672 data: 6 6 5 4 6 5 7 30/30 Read block 89 len 32768 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 10 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 20 20 19 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:43:50.747409273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 115 115 115 T T T 1 020-039 60 60 60 119 119 119 T T T 2 040-059 64 64 64 123 123 123 F F F 3 060-079 66 66 66 125 125 125 F F F 4 080-099 66 66 66 125 125 125 T T T 5 100-119 65 65 65 124 124 124 F F F 6 120-139 62 62 62 121 121 121 T T T 7 140-159 65 65 65 124 124 124 T T T 8 160-179 60 60 60 119 119 119 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 63 63 63 122 122 122 T T T 11 220-239 64 64 64 123 123 123 F F F 12 240-259 59 59 59 117 117 117 T T T 13 260-279 59 59 59 117 117 117 F F F 14 280-299 64 64 64 123 123 123 F F F 15 300-319 61 61 61 120 120 120 F F F 16 320-339 65 65 65 124 124 124 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 66 66 66 125 125 125 F F F 20 400-419 66 66 66 125 125 125 F F F 21 420-439 66 66 66 125 125 125 F F F 22 440-459 65 65 65 124 124 124 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 62 62 62 121 121 121 T T T 25 500-519 66 66 66 125 125 125 F F F 26 520-539 61 61 61 120 120 120 T T T 27 540-559 66 66 66 125 125 125 T T T 28 560-579 62 62 62 121 121 121 F F F 29 580-599 66 66 66 125 125 125 T T T Max gen: 66, Max flush: 125 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 126 126 126 F F F 1 020-039 67 67 67 126 126 126 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 66 66 66 125 125 125 F F F 4 080-099 67 67 67 126 126 126 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 67 67 67 126 126 126 F F F 7 140-159 67 67 67 126 126 126 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 59 59 59 117 117 117 F F F 10 200-219 67 67 67 126 126 126 F F F 11 220-239 64 64 64 123 123 123 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 59 59 59 117 117 117 F F F 14 280-299 64 64 64 123 123 123 F F F 15 300-319 61 61 61 120 120 120 F F F 16 320-339 65 65 65 124 124 124 F F F 17 340-359 48 48 48 101 101 101 F F F 18 360-379 62 62 62 121 121 121 F F F 19 380-399 66 66 66 125 125 125 F F F 20 400-419 66 66 66 125 125 125 F F F 21 420-439 66 66 66 125 125 125 F F F 22 440-459 65 65 65 124 124 124 F F F 23 460-479 50 50 50 103 103 103 F F F 24 480-499 67 67 67 126 126 126 F F F 25 500-519 66 66 66 125 125 125 F F F 26 520-539 67 67 67 126 126 126 F F F 27 540-559 67 67 67 126 126 126 F F F 28 560-579 62 62 62 121 121 121 F F F 29 580-599 67 67 67 126 126 126 F F F Max gen: 67, Max flush: 126 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.018747386Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.0624464Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.063051755Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.063094095Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.063109494Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.063124254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464} {"msg":"Crucible 63470ba2-589a-462e-ba90-61725cbd4c2c has session id: b62c6425-3d24-4ef7-9055-7b79ec5a7652","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.063719219Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Upstairs opts: Upstairs UUID: 63470ba2-589a-462e-ba90-61725cbd4c2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.063751739Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Crucible stats registered with UUID: 63470ba2-589a-462e-ba90-61725cbd4c2c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.063768279Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:05.063781039Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064230655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064260814Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464} {"msg":"63470ba2-589a-462e-ba90-61725cbd4c2c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064302554Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064319064Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"0","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064347533Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"0","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064361683Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"1","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064375633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"1","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064388403Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"2","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064402213Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"2","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064415373Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064444022Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.064471152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"ds_connection connected from Ok(127.0.0.1:62050)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06466034Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"ds_connection connected from Ok(127.0.0.1:56650)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06470228Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"ds_connection connected from Ok(127.0.0.1:35120)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06473235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.065338305Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"0","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.065370184Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"1","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.065386364Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"2","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066006818Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"2","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066038508Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"0","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066097487Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"1","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066663902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066690482Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[0]R flush_numbers[0..12]: [126, 126, 123, 125, 126, 124, 126, 126, 119, 117, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066771992Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[0]R generation[0..12]: [67, 67, 64, 66, 67, 65, 67, 67, 60, 59, 67, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066792751Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066808241Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[1]R flush_numbers[0..12]: [126, 126, 123, 125, 126, 124, 126, 126, 119, 117, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066822411Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[1]R generation[0..12]: [67, 67, 64, 66, 67, 65, 67, 67, 60, 59, 67, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066836741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066851361Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[2]R flush_numbers[0..12]: [126, 126, 123, 125, 126, 124, 126, 126, 119, 117, 126, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066865811Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[2]R generation[0..12]: [67, 67, 64, 66, 67, 65, 67, 67, 60, 59, 67, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066883171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06689864Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06691254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06692636Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06693974Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06695378Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"":"downstairs","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06697292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.06698541Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"0","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.066998539Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"1","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.067011819Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"client":"2","":"downstairs","session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.067024989Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.067037769Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"63470ba2-589a-462e-ba90-61725cbd4c2c is now active with session: b62c6425-3d24-4ef7-9055-7b79ec5a7652","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.067050499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.067062939Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464,"session_id":"b62c6425-3d24-4ef7-9055-7b79ec5a7652"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.067075649Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464} {"msg":"Activated sub_volume 63470ba2-589a-462e-ba90-61725cbd4c2c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:05.067086749Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 577 len 36864 data: 6 7 9 8 7 6 6 7 5 02/30 Read block 376 len 12288 03/30 Write block 355 len 24576 data: 2 4 5 4 4 5 04/30 Write block 575 len 24576 data: 5 5 7 8 10 9 05/30 Read block 450 len 24576 06/30 Read block 440 len 36864 07/30 Read block 190 len 20480 08/30 Write block 89 len 20480 data: 9 9 11 10 9 09/30 Write block 316 len 40960 data: 7 5 6 6 6 7 5 7 7 9 10/30 Read block 353 len 24576 11/30 Read block 159 len 36864 12/30 Read block 499 len 40960 13/30 Write block 280 len 12288 data: 9 9 9 14/30 Write block 198 len 24576 data: 7 7 6 5 5 7 15/30 Write block 476 len 24576 data: 5 6 6 3 5 5 16/30 Read block 427 len 32768 17/30 Read block 74 len 40960 18/30 Write block 271 len 32768 data: 8 7 7 6 5 4 6 8 19/30 Read block 365 len 4096 20/30 Write block 86 len 4096 data: 7 21/30 Write block 394 len 36864 data: 8 8 11 12 9 9 10 10 10 22/30 Read block 483 len 32768 23/30 Flush 24/30 Write block 372 len 32768 data: 7 7 8 8 8 9 9 8 25/30 Write block 406 len 20480 data: 4 3 2 2 2 26/30 Flush 27/30 Write block 418 len 8192 data: 8 7 28/30 Read block 204 len 24576 29/30 Read block 198 len 16384 30/30 Write block 564 len 8192 data: 4 5 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 4 Done Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 3 2 2 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:05.088446785Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 126 126 126 F F F 1 020-039 67 67 67 126 126 126 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 66 66 66 125 125 125 F F F 4 080-099 68 68 68 127 127 127 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 67 67 67 126 126 126 F F F 7 140-159 67 67 67 126 126 126 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 68 68 68 127 127 127 F F F 10 200-219 68 68 68 127 127 127 F F F 11 220-239 64 64 64 123 123 123 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 68 68 68 127 127 127 F F F 14 280-299 68 68 68 127 127 127 F F F 15 300-319 68 68 68 127 127 127 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 68 68 68 127 127 127 F F F 18 360-379 68 68 68 128 128 128 F F F 19 380-399 68 68 68 127 127 127 F F F 20 400-419 68 68 68 128 128 128 T T T 21 420-439 66 66 66 125 125 125 F F F 22 440-459 65 65 65 124 124 124 F F F 23 460-479 68 68 68 127 127 127 F F F 24 480-499 68 68 68 127 127 127 F F F 25 500-519 66 66 66 125 125 125 F F F 26 520-539 67 67 67 126 126 126 F F F 27 540-559 67 67 67 126 126 126 F F F 28 560-579 68 68 68 127 127 127 T T T 29 580-599 68 68 68 127 127 127 F F F Max gen: 68, Max flush: 128 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 67 67 67 126 126 126 F F F 1 020-039 67 67 67 126 126 126 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 66 66 66 125 125 125 F F F 4 080-099 68 68 68 127 127 127 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 67 67 67 126 126 126 F F F 7 140-159 67 67 67 126 126 126 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 68 68 68 127 127 127 F F F 10 200-219 68 68 68 127 127 127 F F F 11 220-239 64 64 64 123 123 123 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 68 68 68 127 127 127 F F F 14 280-299 68 68 68 127 127 127 F F F 15 300-319 68 68 68 127 127 127 F F F 16 320-339 68 68 68 127 127 127 F F F 17 340-359 68 68 68 127 127 127 F F F 18 360-379 68 68 68 128 128 128 F F F 19 380-399 68 68 68 127 127 127 F F F 20 400-419 69 69 69 129 129 129 F F F 21 420-439 66 66 66 125 125 125 F F F 22 440-459 65 65 65 124 124 124 F F F 23 460-479 68 68 68 127 127 127 F F F 24 480-499 68 68 68 127 127 127 F F F 25 500-519 66 66 66 125 125 125 F F F 26 520-539 67 67 67 126 126 126 F F F 27 540-559 67 67 67 126 126 126 F F F 28 560-579 69 69 69 129 129 129 F F F 29 580-599 68 68 68 127 127 127 F F F Max gen: 69, Max flush: 129 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.30984976Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.353730493Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.354438137Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.354490917Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.354508326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.354526866Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472} {"msg":"Crucible c4096b97-8180-46e6-88c5-5273fd965a7f has session id: c383258a-49d8-4347-99d7-e895d12e75a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.35516804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Upstairs opts: Upstairs UUID: c4096b97-8180-46e6-88c5-5273fd965a7f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.3552117Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Crucible stats registered with UUID: c4096b97-8180-46e6-88c5-5273fd965a7f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.35523675Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:09.355257639Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.355662226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.355697526Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472} {"msg":"c4096b97-8180-46e6-88c5-5273fd965a7f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.355721545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.355745085Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"0","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.355792445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"0","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.355823754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"1","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.355855304Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"1","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.355997603Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"2","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356042923Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"2","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356068472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356141311Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356180811Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:50166)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356205591Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49549)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356228961Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:53042)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356252221Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356774336Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"2","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.356814076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"0","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.35741325Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"0","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.357565529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"2","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.357786467Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"1","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.358127504Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.358275802Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.358442031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"1","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[0]R flush_numbers[0..12]: [126, 126, 123, 125, 127, 124, 126, 126, 119, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.358965286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[0]R generation[0..12]: [67, 67, 64, 66, 68, 65, 67, 67, 60, 68, 68, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.358993636Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359015876Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[1]R flush_numbers[0..12]: [126, 126, 123, 125, 127, 124, 126, 126, 119, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359030715Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[1]R generation[0..12]: [67, 67, 64, 66, 68, 65, 67, 67, 60, 68, 68, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359044355Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359058825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[2]R flush_numbers[0..12]: [126, 126, 123, 125, 127, 124, 126, 126, 119, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359072565Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[2]R generation[0..12]: [67, 67, 64, 66, 68, 65, 67, 67, 60, 68, 68, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359085745Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359099225Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359111835Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359124704Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":Wait for a query_work_queue command to finish before sending IO "downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359145654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359158474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"":"downstairs","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359171234Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359182914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"0","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359195894Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"1","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359211484Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"client":"2","":"downstairs","session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359225854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359238054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"c4096b97-8180-46e6-88c5-5273fd965a7f is now active with session: c383258a-49d8-4347-99d7-e895d12e75a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359250803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359262543Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472,"session_id":"c383258a-49d8-4347-99d7-e895d12e75a6"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359274493Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472} {"msg":"Activated sub_volume c4096b97-8180-46e6-88c5-5273fd965a7f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:09.359285473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 32768 data: 6 7 7 7 6 4 4 2 02/30 Write block 385 len 20480 data: 7 6 7 8 8 03/30 Read block 30 len 16384 04/30 Read block 143 len 20480 05/30 Write block 8 len 24576 data: 9 8 9 10 8 6 06/30 Write block 24 len 8192 data: 2 3 07/30 Write block 569 len 28672 data: 6 6 7 6 4 5 6 08/30 Write block 13 len 32768 data: 7 5 6 7 8 8 8 7 09/30 Write block 344 len 24576 data: 3 4 3 3 3 3 10/30 Read block 169 len 36864 11/30 Flush 12/30 Write block 133 len 12288 data: 4 5 5 13/30 Read block 501 len 20480 14/30 Write block 552 len 8192 data: 6 7 15/30 Read block 588 len 40960 16/30 Write block 318 len 28672 data: 7 7 7 8 6 8 8 17/30 Flush 18/30 Write block 529 len 12288 data: 9 7 4 19/30 Read block 557 len 4096 20/30 Read block 298 len 16384 21/30 Read block 398 len 32768 22/30 Read block 433 len 40960 23/30 Read block 527 len 36864 24/30 Read block 347 len 16384 25/30 Write block 503 len 4096 data: 6 26/30 Read block 458 len 12288 27/30 Read block 112 len 28672 28/30 Write block 444 len 4096 data: 8 29/30 Write block 577 len 12288 data: 8 9 11 30/30 Read block 379 len 20480 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 13 11 11 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:09.378482079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 70 70 70 130 130 130 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 66 66 66 125 125 125 F F F 4 080-099 68 68 68 127 127 127 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 70 70 70 131 131 131 F F F 7 140-159 67 67 67 126 126 126 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 68 68 68 127 127 127 F F F 10 200-219 68 68 68 127 127 127 F F F 11 220-239 64 64 64 123 123 123 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 68 68 68 127 127 127 F F F 14 280-299 68 68 68 127 127 127 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 70 70 70 131 131 131 F F F 17 340-359 70 70 70 130 130 130 F F F 18 360-379 68 68 68 128 128 128 F F F 19 380-399 70 70 70 130 130 130 F F F 20 400-419 69 69 69 129 129 129 F F F 21 420-439 66 66 66 125 125 125 F F F 22 440-459 65 65 65 124 124 124 T T T 23 460-479 68 68 68 127 127 127 F F F 24 480-499 68 68 68 127 127 127 F F F 25 500-519 66 66 66 125 125 125 T T T 26 520-539 67 67 67 126 126 126 T T T 27 540-559 70 70 70 131 131 131 F F F 28 560-579 70 70 70 130 130 130 T T T 29 580-599 68 68 68 127 127 127 F F F Max gen: 70, Max flush: 131 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 70 70 70 130 130 130 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 66 66 66 125 125 125 F F F 4 080-099 68 68 68 127 127 127 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 70 70 70 131 131 131 F F F 7 140-159 67 67 67 126 126 126 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 68 68 68 127 127 127 F F F 10 200-219 68 68 68 127 127 127 F F F 11 220-239 64 64 64 123 123 123 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 68 68 68 127 127 127 F F F 14 280-299 68 68 68 127 127 127 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 70 70 70 131 131 131 F F F 17 340-359 70 70 70 130 130 130 F F F 18 360-379 68 68 68 128 128 128 F F F 19 380-399 70 70 70 130 130 130 F F F 20 400-419 69 69 69 129 129 129 F F F 21 420-439 66 66 66 125 125 125 F F F 22 440-459 71 71 71 132 132 132 F F F 23 460-479 68 68 68 127 127 127 F F F 24 480-499 68 68 68 127 127 127 F F F 25 500-519 71 71 71 132 132 132 F F F 26 520-539 71 71 71 132 132 132 F F F 27 540-559 70 70 70 131 131 131 F F F 28 560-579 71 71 71 132 132 132 F F F 29 580-599 68 68 68 127 127 127 F F F Max gen: 71, Max flush: 132 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.612805247Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.656504632Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.657094606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.657139416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.657154366Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.657168646Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480} {"msg":"Crucible ae1d2334-7b41-4114-811a-ece0d2221e6c has session id: acc3cf0b-daf6-42ec-a817-90d200fe093d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.657705071Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Upstairs opts: Upstairs UUID: ae1d2334-7b41-4114-811a-ece0d2221e6c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.65783079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Crucible stats registered with UUID: ae1d2334-7b41-4114-811a-ece0d2221e6c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.657866949Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:13.657889339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658359485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658386585Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480} {"msg":"ae1d2334-7b41-4114-811a-ece0d2221e6c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658439504Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658465704Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"0","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658498674Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"0","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658513143Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"1","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658538363Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"1","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658564093Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"2","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658580043Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"2","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658596073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658620342Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658662662Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60997)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658690752Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47613)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.658719102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34164)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.65886887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.659366846Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"0","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.659408395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"1","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.660059529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"1","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.660191808Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"0","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.660287358Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"2","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.660688344Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.660790343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661061151Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"2","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[0]R flush_numbers[0..12]: [130, 130, 123, 125, 127, 124, 131, 126, 119, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661608455Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[0]R generation[0..12]: [70, 70, 64, 66, 68, 65, 70, 67, 60, 68, 68, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661630015Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661645135Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[1]R flush_numbers[0..12]: [130, 130, 123, 125, 127, 124, 131, 126, 119, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661659175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[1]R generation[0..12]: [70, 70, 64, 66, 68, 65, 70, 67, 60, 68, 68, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661672565Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661686295Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[2]R flush_numbers[0..12]: [130, 130, 123, 125, 127, 124, 131, 126, 119, 127, 127, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661699765Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[2]R generation[0..12]: [70, 70, 64, 66, 68, 65, 70, 67, 60, 68, 68, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661712985Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661726554Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661742714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661756214Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661768914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661781434Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"":"downstairs","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661794424Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661805974Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"0","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661818764Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"1","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661831413Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"client":"2","":"downstairs","session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661844373Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661856173Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Wait for a query_work_queue command to finish before sending IO ae1d2334-7b41-4114-811a-ece0d2221e6c is now active with session: acc3cf0b-daf6-42ec-a817-90d200fe093d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661874403Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661886303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480,"session_id":"acc3cf0b-daf6-42ec-a817-90d200fe093d"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661898343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480} {"msg":"Activated sub_volume ae1d2334-7b41-4114-811a-ece0d2221e6c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:13.661909133Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 478 len 36864 data: 7 4 6 6 4 5 6 6 7 02/30 Write block 94 len 8192 data: 7 6 03/30 Flush 04/30 Write block 565 len 20480 data: 6 4 6 6 7 05/30 Write block 202 len 20480 data: 6 8 8 6 7 06/30 Read block 217 len 40960 07/30 Write block 273 len 24576 data: 8 7 6 5 7 9 08/30 Read block 106 len 24576 09/30 Write block 517 len 4096 data: 10 10/30 Read block 480 len 28672 11/30 Read block 429 len 28672 12/30 Read block 436 len 40960 13/30 Write block 136 len 40960 data: 5 8 6 5 6 5 5 4 3 4 14/30 Write block 288 len 32768 data: 4 5 7 7 6 5 6 6 15/30 Write block 508 len 40960 data: 6 6 6 6 6 7 7 7 9 11 16/30 Write block 337 len 24576 data: 8 7 6 6 5 5 17/30 Write block 569 len 4096 data: 8 18/30 Write block 82 len 8192 data: 7 7 19/30 Read block 291 len 40960 20/30 Read block 141 len 4096 21/30 Read block 415 len 32768 22/30 Read block 465 len 4096 23/30 Write block 397 len 12288 data: 13 10 10 24/30 Read block 0 len 12288 25/30 Write block 432 len 16384 data: 6 4 4 5 26/30 Write block 65 len 24576 data: 12 11 10 11 10 7 27/30 Read block 462 len 28672 28/30 Write block 551 len 32768 data: 9 7 8 6 7 6 7 4 29/30 Write block 372 len 12288 data: 8 8 9 30/30 Write block 388 len 24576 data: 9 9 9 8 9 8 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 5 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:13.680664173Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 70 70 70 130 130 130 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 66 66 66 125 125 125 T T T 4 080-099 72 72 72 133 133 133 T T T 5 100-119 65 65 65 124 124 124 F F F 6 120-139 70 70 70 131 131 131 T T T 7 140-159 67 67 67 126 126 126 T T T 8 160-179 60 60 60 119 119 119 F F F 9 180-199 68 68 68 127 127 127 F F F 10 200-219 68 68 68 127 127 127 T T T 11 220-239 64 64 64 123 123 123 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 68 68 68 127 127 127 T T T 14 280-299 68 68 68 127 127 127 T T T 15 300-319 70 70 70 131 131 131 F F F 16 320-339 70 70 70 131 131 131 T T T 17 340-359 70 70 70 130 130 130 T T T 18 360-379 68 68 68 128 128 128 T T T 19 380-399 70 70 70 130 130 130 T T T 20 400-419 69 69 69 129 129 129 F F F 21 420-439 66 66 66 125 125 125 T T T 22 440-459 71 71 71 132 132 132 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 133 133 133 F F F 25 500-519 71 71 71 132 132 132 T T T 26 520-539 71 71 71 132 132 132 F F F 27 540-559 70 70 70 131 131 131 T T T 28 560-579 71 71 71 132 132 132 T T T 29 580-599 68 68 68 127 127 127 F F F Max gen: 72, Max flush: 133 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 70 70 70 130 130 130 F F F 2 040-059 64 64 64 123 123 123 F F F 3 060-079 73 73 73 134 134 134 F F F 4 080-099 73 73 73 134 134 134 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 73 73 73 134 134 134 F F F 7 140-159 73 73 73 134 134 134 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 68 68 68 127 127 127 F F F 10 200-219 73 73 73 134 134 134 F F F 11 220-239 64 64 64 123 123 123 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 73 73 73 134 134 134 F F F 14 280-299 73 73 73 134 134 134 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 73 73 73 134 134 134 F F F 17 340-359 73 73 73 134 134 134 F F F 18 360-379 73 73 73 134 134 134 F F F 19 380-399 73 73 73 134 134 134 F F F 20 400-419 69 69 69 129 129 129 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 71 71 71 132 132 132 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 133 133 133 F F F 25 500-519 73 73 73 134 134 134 F F F 26 520-539 71 71 71 132 132 132 F F F 27 540-559 73 73 73 134 134 134 F F F 28 560-579 73 73 73 134 134 134 F F F 29 580-599 68 68 68 127 127 127 F F F Max gen: 73, Max flush: 134 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:17.972658359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:18.016843029Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.017235636Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.048948369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.049654232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.049689692Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.049703362Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.049717332Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488} {"msg":"Crucible 8bf47757-927d-4ccf-96a4-4a6a067ebb52 has session id: 055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.050337276Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Upstairs opts: Upstairs UUID: 8bf47757-927d-4ccf-96a4-4a6a067ebb52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.050381396Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Crucible stats registered with UUID: 8bf47757-927d-4ccf-96a4-4a6a067ebb52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.050401456Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:18.050417945Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.050899601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.050929031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488} {"msg":"8bf47757-927d-4ccf-96a4-4a6a067ebb52 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.05097892Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.05099865Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"0","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.05103387Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"0","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.05105129Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"1","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.05106861Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"1","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051088759Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"2","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051106389Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"2","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051122819Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051139909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051156809Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51145)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051173669Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45486)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051356377Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:60268)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051492736Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.051973651Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"1","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.052019401Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"2","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.052305278Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"0","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.052876493Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"1","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.052916963Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"2","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.052947562Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"0","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053479248Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053610487Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[0]R flush_numbers[0..12]: [130, 130, 123, 134, 134, 124, 134, 134, 119, 127, 134, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053687406Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[0]R generation[0..12]: [70, 70, 64, 73, 73, 65, 73, 73, 60, 68, 73, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053709055Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053724286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[1]R flush_numbers[0..12]: [130, 130, 123, 134, 134, 124, 134, 134, 119, 127, 134, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053738235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[1]R generation[0..12]: [70, 70, 64, 73, 73, 65, 73, 73, 60, 68, 73, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053751955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053766555Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[2]R flush_numbers[0..12]: [130, 130, 123, 134, 134, 124, 134, 134, 119, 127, 134, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053780155Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[2]R generation[0..12]: [70, 70, 64, 73, 73, 65, 73, 73, 60, 68, 73, 64]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053793395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053806785Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053819294Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053832065Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053844544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053860254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"":"downstairs","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053873674Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053885504Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"0","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053898534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"1","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053911454Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"client":"2","":"downstairs","session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:44:18.053924404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053942284Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"8bf47757-927d-4ccf-96a4-4a6a067ebb52 is now active with session: 055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053955053Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053966713Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488,"session_id":"055e1f7e-6f18-400a-ac4a-7c3f65bf1c8c"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053978513Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488} {"msg":"Activated sub_volume 8bf47757-927d-4ccf-96a4-4a6a067ebb52","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:18.053989383Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 521 len 8192 data: 9 6 02/30 Write block 229 len 40960 data: 8 7 6 5 8 10 11 8 7 8 03/30 Write block 519 len 36864 data: 9 8 10 7 5 6 6 8 8 04/30 Flush 05/30 Write block 55 len 12288 data: 8 9 6 06/30 Read block 290 len 16384 07/30 Read block 59 len 40960 08/30 Write block 552 len 24576 data: 8 9 7 8 7 8 09/30 Read block 318 len 8192 10/30 Read block 173 len 40960 11/30 Read block 447 len 28672 12/30 Write block 120 len 32768 data: 8 9 9 10 8 6 6 6 13/30 Read block 551 len 16384 14/30 Write block 291 len 16384 data: 8 7 6 7 15/30 Flush 16/30 Write block 388 len 24576 data: 10 10 10 9 10 9 17/30 Read block 126 len 40960 18/30 Write block 378 len 12288 data: 10 9 7 19/30 Read block 48 len 8192 20/30 Read block 220 len 40960 21/30 Write block 197 len 28672 data: 6 8 8 7 6 7 9 22/30 Write block 286 len 20480 data: 5 5 5 6 8 23/30 Write block 389 len 24576 data: 11 11 10 11 10 9 24/30 Flush 25/30 Write block 341 len 40960 data: 6 6 5 4 5 4 4 4 4 4 26/30 Read block 299 len 36864 27/30 Read block 392 len 24576 28/30 Write block 33 len 8192 data: 6 6 29/30 Write block 410 len 24576 data: 3 5 6 6 6 6 30/30 Read block 446 len 28672 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:18.075788786Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 70 70 70 130 130 130 T T T 2 040-059 74 74 74 136 136 136 F F F 3 060-079 73 73 73 134 134 134 F F F 4 080-099 73 73 73 134 134 134 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 74 74 74 136 136 136 F F F 7 140-159 73 73 73 134 134 134 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 74 74 74 137 137 137 F F F 10 200-219 74 74 74 137 137 137 F F F 11 220-239 74 74 74 135 135 135 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 73 73 73 134 134 134 F F F 14 280-299 74 74 74 137 137 137 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 73 73 73 134 134 134 F F F 17 340-359 73 73 73 134 134 134 T T T 18 360-379 74 74 74 137 137 137 F F F 19 380-399 74 74 74 137 137 137 F F F 20 400-419 69 69 69 129 129 129 T T T 21 420-439 73 73 73 134 134 134 F F F 22 440-459 71 71 71 132 132 132 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 133 133 133 F F F 25 500-519 74 74 74 135 135 135 F F F 26 520-539 74 74 74 135 135 135 F F F 27 540-559 74 74 74 136 136 136 F F F 28 560-579 73 73 73 134 134 134 F F F 29 580-599 68 68 68 127 127 127 F F F Max gen: 74, Max flush: 137 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 75 75 75 138 138 138 F F F 2 040-059 74 74 74 136 136 136 F F F 3 060-079 73 73 73 134 134 134 F F F 4 080-099 73 73 73 134 134 134 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 74 74 74 136 136 136 F F F 7 140-159 73 73 73 134 134 134 F F F 8 160-179 60 60 60 119 119 119 F F F 9 180-199 74 74 74 137 137 137 F F F 10 200-219 74 74 74 137 137 137 F F F 11 220-239 74 74 74 135 135 135 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 73 73 73 134 134 134 F F F 14 280-299 74 74 74 137 137 137 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 73 73 73 134 134 134 F F F 17 340-359 75 75 75 138 138 138 F F F 18 360-379 74 74 74 137 137 137 F F F 19 380-399 74 74 74 137 137 137 F F F 20 400-419 75 75 75 138 138 138 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 71 71 71 132 132 132 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 133 133 133 F F F 25 500-519 74 74 74 135 135 135 F F F 26 520-539 74 74 74 135 135 135 F F F 27 540-559 74 74 74 136 136 136 F F F 28 560-579 73 73 73 134 134 134 F F F 29 580-599 68 68 68 127 127 127 F F F Max gen: 75, Max flush: 138 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.334761911Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.378846052Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.379735834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.379786804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.379804673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.379822633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496} {"msg":"Crucible f622a519-a136-419a-9beb-3e1fbb0251aa has session id: 15cab559-20f4-4471-a35b-2b59f620ce71","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.380497097Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Upstairs opts: Upstairs UUID: f622a519-a136-419a-9beb-3e1fbb0251aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.380538837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Crucible stats registered with UUID: f622a519-a136-419a-9beb-3e1fbb0251aa","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.380557937Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:22.380570966Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381095872Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381120892Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496} {"msg":"f622a519-a136-419a-9beb-3e1fbb0251aa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381196941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.38121472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"0","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.38124513Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"0","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.3812591Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"1","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.38127543Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"1","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.38129879Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"2","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381356089Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"2","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381383069Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381409239Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381435198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"ds_connection connected from Ok(127.0.0.1:34630)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381462788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"ds_connection connected from Ok(127.0.0.1:40397)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381489778Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"ds_connection connected from Ok(127.0.0.1:43010)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.381517058Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.382040853Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"2","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.382195842Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"0","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.382616628Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"2","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.382761906Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"0","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.383224012Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.383359371Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.383571729Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"1","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384142854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"1","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[0]R flush_numbers[0..12]: [130, 138, 136, 134, 134, 124, 136, 134, 119, 137, 137, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384663079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[0]R generation[0..12]: [70, 75, 74, 73, 73, 65, 74, 73, 60, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384685519Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384700929Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[1]R flush_numbers[0..12]: [130, 138, 136, 134, 134, 124, 136, 134, 119, 137, 137, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384715269Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[1]R generation[0..12]: [70, 75, 74, 73, 73, 65, 74, 73, 60, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384729079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384743499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[2]R flush_numbers[0..12]: [130, 138, 136, 134, 134, 124, 136, 134, 119, 137, 137, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384761428Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[2]R generation[0..12]: [70, 75, 74, 73, 73, 65, 74, 73, 60, 74, 74, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384775758Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384790218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384803328Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384816008Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384829028Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384848208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"":"downstairs","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384861107Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384872827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"0","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384886157Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"1","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384899677Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"client":"2","":"downstairs","session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384912497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384925247Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"f622a519-a136-419a-9beb-3e1fbb0251aa is now active with session: 15cab559-20f4-4471-a35b-2b59f620ce71","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384938517Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384950147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496,"session_id":"15cab559-20f4-4471-a35b-2b59f620ce71"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384972497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496} {"msg":"Activated sub_volume f622a519-a136-419a-9beb-3e1fbb0251aa","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:22.384986166Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 455 len 16384 data: 6 6 6 6 02/30 Write block 162 len 36864 data: 4 4 3 2 3 2 3 3 3 03/30 Read block 228 len 8192 04/30 Read block 325 len 28672 05/30 Write block 505 len 36864 data: 5 6 8 7 7 7 7 7 8 06/30 Read block 388 len 12288 07/30 Write block 181 len 12288 data: 5 5 4 08/30 Read block 222 len 12288 09/30 Write block 593 len 16384 data: 7 6 6 5 10/30 Read block 319 len 4096 11/30 Write block 593 len 28672 data: 8 7 7 6 3 3 2 12/30 Write block 65 len 8192 data: 13 12 13/30 Write block 400 len 4096 data: 11 14/30 Write block 331 len 28672 data: 10 6 5 7 8 7 9 15/30 Read block 540 len 16384 16/30 Write block 388 len 40960 data: 11 12 12 11 12 11 10 9 12 14 17/30 Read block 263 len 20480 18/30 Read block 276 len 40960 19/30 Write block 524 len 8192 data: 7 7 20/30 Write block 381 len 8192 data: 5 6 21/30 Write block 332 len 40960 data: 7 6 8 9 8 10 8 7 7 7 22/30 Read block 104 len 4096 23/30 Read block 210 len 24576 24/30 Read block 285 len 24576 25/30 Flush 26/30 Write block 209 len 20480 data: 11 9 8 7 7 27/30 Write block 547 len 8192 data: 8 9 28/30 Read block 561 len 8192 29/30 Write block 565 len 8192 data: 7 5 30/30 Write block 50 len 36864 data: 6 7 9 6 8 9 10 7 4 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:22.404578379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 75 75 75 138 138 138 F F F 2 040-059 74 74 74 136 136 136 T T T 3 060-079 76 76 76 139 139 139 F F F 4 080-099 73 73 73 134 134 134 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 74 74 74 136 136 136 F F F 7 140-159 73 73 73 134 134 134 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 76 76 76 139 139 139 F F F 10 200-219 74 74 74 137 137 137 T T T 11 220-239 74 74 74 135 135 135 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 73 73 73 134 134 134 F F F 14 280-299 74 74 74 137 137 137 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 76 76 76 139 139 139 F F F 17 340-359 76 76 76 139 139 139 F F F 18 360-379 74 74 74 137 137 137 F F F 19 380-399 76 76 76 139 139 139 F F F 20 400-419 76 76 76 139 139 139 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 76 76 76 139 139 139 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 133 133 133 F F F 25 500-519 76 76 76 139 139 139 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 74 74 74 136 136 136 T T T 28 560-579 73 73 73 134 134 134 T T T 29 580-599 76 76 76 139 139 139 F F F Max gen: 76, Max flush: 139 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 75 75 75 138 138 138 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 76 76 76 139 139 139 F F F 4 080-099 73 73 73 134 134 134 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 74 74 74 136 136 136 F F F 7 140-159 73 73 73 134 134 134 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 76 76 76 139 139 139 F F F 10 200-219 77 77 77 140 140 140 F F F 11 220-239 74 74 74 135 135 135 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 73 73 73 134 134 134 F F F 14 280-299 74 74 74 137 137 137 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 76 76 76 139 139 139 F F F 17 340-359 76 76 76 139 139 139 F F F 18 360-379 74 74 74 137 137 137 F F F 19 380-399 76 76 76 139 139 139 F F F 20 400-419 76 76 76 139 139 139 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 76 76 76 139 139 139 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 133 133 133 F F F 25 500-519 76 76 76 139 139 139 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 77 77 77 140 140 140 F F F 28 560-579 77 77 77 140 140 140 F F F 29 580-599 76 76 76 139 139 139 F F F Max gen: 77, Max flush: 140 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.63741493Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.681383213Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.682057447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.682101226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.682116336Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.682136366Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504} {"msg":"Crucible 7f6351a4-a67f-4c7d-aa4c-db3630dd3ac9 has session id: 15ebae6f-20bb-43fe-bb10-fc6be61abfb8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.682607882Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Upstairs opts: Upstairs UUID: 7f6351a4-a67f-4c7d-aa4c-db3630dd3ac9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.682647131Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Crucible stats registered with UUID: 7f6351a4-a67f-4c7d-aa4c-db3630dd3ac9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.682670251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:26.682688081Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683098597Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683123957Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504} {"msg":"7f6351a4-a67f-4c7d-aa4c-db3630dd3ac9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683189366Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683207466Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"0","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683236846Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"0","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683251326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"1","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683265086Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"1","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683278395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"2","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683302315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"2","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683332515Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683358625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683384765Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43216)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683408554Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:59244)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683432164Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"ds_connection connected from Ok(127.0.0.1:44608)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.683456864Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.684156788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"0","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.684185727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"2","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.684802192Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"0","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.684926291Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"2","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.685529385Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.685563295Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.686207599Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"1","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.686706124Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"1","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[0]R flush_numbers[0..12]: [130, 138, 140, 139, 134, 124, 136, 134, 139, 139, 140, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[0]R generation[0..12]: [70, 75, 77, 76, 73, 65, 74, 73, 76, 76, 77, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687265379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687284349Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[1]R flush_numbers[0..12]: [130, 138, 140, 139, 134, 124, 136, 134, 139, 139, 140, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687301859Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[1]R generation[0..12]: [70, 75, 77, 76, 73, 65, 74, 73, 76, 76, 77, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687318339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687335599Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[2]R flush_numbers[0..12]: [130, 138, 140, 139, 134, 124, 136, 134, 139, 139, 140, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687352679Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[2]R generation[0..12]: [70, 75, 77, 76, 73, 65, 74, 73, 76, 76, 77, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687369558Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:44:26.687386729Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687405788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687421908Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687437498Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687453098Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"":"downstairs","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687468878Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687483328Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"0","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687502797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"1","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687518877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"client":"2","":"downstairs","session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687534937Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687549637Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"7f6351a4-a67f-4c7d-aa4c-db3630dd3ac9 is now active with session: 15ebae6f-20bb-43fe-bb10-fc6be61abfb8","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687565197Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687579747Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504,"session_id":"15ebae6f-20bb-43fe-bb10-fc6be61abfb8"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:26.687594636Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504} {"msg":"Activated sub_volume 7f6351a4-a67f-4c7d-aa4c-db3630dd3ac9","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T01:44:26.687608156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 74 len 28672 data: 6 5 7 8 8 7 8 02/30 Write block 350 len 16384 data: 5 4 4 3 03/30 Read block 5 len 32768 04/30 Write block 85 len 4096 data: 5 05/30 Read block 228 len 28672 06/30 Read block 101 len 28672 07/30 Write block 27 len 28672 data: 7 9 7 7 7 8 7 08/30 Write block 206 len 20480 data: 8 10 10 12 10 09/30 Read block 283 len 28672 10/30 Flush 11/30 Write block 206 len 16384 data: 9 11 11 13 12/30 Read block 189 len 36864 13/30 Read block 374 len 8192 14/30 Read block 54 len 4096 15/30 Read block 561 len 20480 16/30 Read block 445 len 28672 17/30 Read block 512 len 8192 18/30 Read block 85 len 4096 19/30 Write block 66 len 4096 data: 13 20/30 Read block 470 len 32768 21/30 Read block 501 len 12288 22/30 Write block 189 len 20480 data: 4 4 4 5 5 23/30 Flush 24/30 Write block 501 len 36864 data: 5 4 7 7 6 7 9 8 8 25/30 Write block 390 len 36864 data: 13 12 13 12 11 10 13 15 11 26/30 Read block 410 len 36864 27/30 Read block 580 len 36864 28/30 Write block 397 len 12288 data: 16 12 11 29/30 Read block 406 len 8192 30/30 Read block 569 len 4096 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:26.707289308Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 142 142 142 F F F 4 080-099 78 78 78 141 141 141 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 74 74 74 136 136 136 F F F 7 140-159 73 73 73 134 134 134 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 74 74 74 135 135 135 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 73 73 73 134 134 134 F F F 14 280-299 74 74 74 137 137 137 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 76 76 76 139 139 139 F F F 17 340-359 78 78 78 141 141 141 F F F 18 360-379 74 74 74 137 137 137 F F F 19 380-399 76 76 76 139 139 139 T T T 20 400-419 76 76 76 139 139 139 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 76 76 76 139 139 139 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 133 133 133 F F F 25 500-519 76 76 76 139 139 139 T T T 26 520-539 76 76 76 139 139 139 F F F 27 540-559 77 77 77 140 140 140 F F F 28 560-579 77 77 77 140 140 140 F F F 29 580-599 76 76 76 139 139 139 F F F Max gen: 78, Max flush: 142 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 130 130 130 F F F 1 020-039 78 78 78 141 141 141 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 142 142 142 F F F 4 080-099 78 78 78 141 141 141 F F F 5 100-119 65 65 65 124 124 124 F F F 6 120-139 74 74 74 136 136 136 F F F 7 140-159 73 73 73 134 134 134 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 74 74 74 135 135 135 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 73 73 73 134 134 134 F F F 14 280-299 74 74 74 137 137 137 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 76 76 76 139 139 139 F F F 17 340-359 78 78 78 141 141 141 F F F 18 360-379 74 74 74 137 137 137 F F F 19 380-399 79 79 79 143 143 143 F F F 20 400-419 76 76 76 139 139 139 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 76 76 76 139 139 139 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 72 72 72 133 133 133 F F F 25 500-519 79 79 79 143 143 143 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 77 77 77 140 140 140 F F F 28 560-579 77 77 77 140 140 140 F F F 29 580-599 76 76 76 139 139 139 F F F Max gen: 79, Max flush: 143 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.929203798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97323606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.973890864Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.973946444Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.973964493Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.973983243Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512} {"msg":"Crucible abcd260c-3fa3-4cab-bf25-d87f54327b50 has session id: 3a6a17bd-f736-4ba5-abfa-b421aeed37c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.974540848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Upstairs opts: Upstairs UUID: abcd260c-3fa3-4cab-bf25-d87f54327b50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.974569448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Crucible stats registered with UUID: abcd260c-3fa3-4cab-bf25-d87f54327b50","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.974584158Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:30.974597538Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975067223Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975093553Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512} {"msg":"abcd260c-3fa3-4cab-bf25-d87f54327b50 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975168152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975188722Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"0","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975236642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"0","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975264721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"1","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975289781Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"1","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975312491Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"2","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975338281Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"2","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975361341Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975384611Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97540448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47390)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97542221Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49826)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975602178Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41516)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.975638878Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.976069504Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"0","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.976096754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"1","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.976820138Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"0","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.976862697Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"1","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.977342403Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.977453532Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.977530721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"2","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978032147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"2","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[0]R flush_numbers[0..12]: [130, 141, 140, 142, 141, 124, 136, 134, 139, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978540962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[0]R generation[0..12]: [70, 78, 77, 78, 78, 65, 74, 73, 76, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978559742Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978574031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[1]R flush_numbers[0..12]: [130, 141, 140, 142, 141, 124, 136, 134, 139, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978587382Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[1]R generation[0..12]: [70, 78, 77, 78, 78, 65, 74, 73, 76, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978600641Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978614111Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[2]R flush_numbers[0..12]: [130, 141, 140, 142, 141, 124, 136, 134, 139, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978628381Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[2]R generation[0..12]: [70, 78, 77, 78, 78, 65, 74, 73, 76, 78, 78, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978641601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978655151Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978671691Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978684741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97869727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid"Wait for a query_work_queue command to finish before sending IO :1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97871689Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"":"downstairs","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97872959Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97874114Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"0","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97875409Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"1","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97876678Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"client":"2","":"downstairs","session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97877943Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97879128Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"abcd260c-3fa3-4cab-bf25-d87f54327b50 is now active with session: 3a6a17bd-f736-4ba5-abfa-b421aeed37c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.97880362Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978815149Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512,"session_id":"3a6a17bd-f736-4ba5-abfa-b421aeed37c2"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978826989Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512} {"msg":"Activated sub_volume abcd260c-3fa3-4cab-bf25-d87f54327b50","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:30.978837909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 81 len 36864 data: 9 8 8 6 6 8 6 8 10 02/30 Write block 153 len 12288 data: 5 6 8 03/30 Write block 15 len 32768 data: 7 8 9 9 9 8 5 5 04/30 Flush 05/30 Write block 483 len 4096 data: 6 06/30 Read block 507 len 12288 07/30 Write block 328 len 40960 data: 9 10 10 11 8 7 9 10 9 11 08/30 Read block 195 len 24576 09/30 Write block 396 len 36864 data: 14 17 13 12 12 11 11 10 6 10/30 Read block 347 len 40960 11/30 Write block 253 len 8192 data: 8 8 12/30 Read block 96 len 8192 13/30 Write block 305 len 20480 data: 5 5 5 6 8 14/30 Write block 282 len 32768 data: 10 9 8 5 6 6 6 7 15/30 Read block 483 len 16384 16/30 Write block 359 len 12288 data: 5 6 4 17/30 Write block 577 len 40960 data: 9 10 12 10 8 7 7 8 6 4 18/30 Write block 586 len 32768 data: 5 7 6 4 5 5 7 9 19/30 Write block 357 len 20480 data: 6 5 6 7 5 20/30 Read block 493 len 36864 21/30 Write block 252 len 36864 data: 8 9 9 8 8 9 8 5 5 22/30 Write block 493 len 40960 data: 6 6 5 3 4 4 4 5 6 5 23/30 Write block 282 len 12288 data: 11 10 9 24/30 Write block 105 len 40960 data: 7 8 7 7 6 8 8 6 7 5 25/30 Flush 26/30 Write block 282 len 4096 data: 12 27/30 Read block 32 len 36864 28/30 Write block 238 len 12288 data: 9 7 10 29/30 Write block 332 len 16384 data: 9 8 10 11 30/30 Write block 0 len 36864 data: 2 3 3 3 4 5 7 10 10 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 4 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:30.998399312Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 144 144 144 T T T 1 020-039 80 80 80 144 144 144 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 142 142 142 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 80 80 80 145 145 145 F F F 6 120-139 74 74 74 136 136 136 F F F 7 140-159 80 80 80 144 144 144 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 74 74 74 135 135 135 T T T 12 240-259 80 80 80 145 145 145 T T T 13 260-279 80 80 80 145 145 145 F F F 14 280-299 80 80 80 145 145 145 T T T 15 300-319 80 80 80 145 145 145 F F F 16 320-339 80 80 80 145 145 145 T T T 17 340-359 80 80 80 145 145 145 F F F 18 360-379 80 80 80 145 145 145 F F F 19 380-399 80 80 80 145 145 145 F F F 20 400-419 80 80 80 145 145 145 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 76 76 76 139 139 139 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 80 80 80 145 145 145 F F F 25 500-519 80 80 80 145 145 145 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 77 77 77 140 140 140 F F F 28 560-579 80 80 80 145 145 145 F F F 29 580-599 80 80 80 145 145 145 F F F Max gen: 80, Max flush: 145 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 146 146 146 F F F 1 020-039 80 80 80 144 144 144 F F F 2 040-059 77 77 77 140 140 140 F F F 3 060-079 78 78 78 142 142 142 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 80 80 80 145 145 145 F F F 6 120-139 74 74 74 136 136 136 F F F 7 140-159 80 80 80 144 144 144 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 81 81 81 146 146 146 F F F 12 240-259 81 81 81 146 146 146 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 81 81 81 146 146 146 F F F 15 300-319 80 80 80 145 145 145 F F F 16 320-339 81 81 81 146 146 146 F F F 17 340-359 80 80 80 145 145 145 F F F 18 360-379 80 80 80 145 145 145 F F F 19 380-399 80 80 80 145 145 145 F F F 20 400-419 80 80 80 145 145 145 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 76 76 76 139 139 139 F F F 23 460-479 72 72 72 133 133 133 F F F 24 480-499 80 80 80 145 145 145 F F F 25 500-519 80 80 80 145 145 145 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 77 77 77 140 140 140 F F F 28 560-579 80 80 80 145 145 145 F F F 29 580-599 80 80 80 145 145 145 F F F Max gen: 81, Max flush: 146 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.235732323Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.280360419Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.280973514Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.281017333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.281032143Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.281046953Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520} {"msg":"Crucible 7feeb4a4-956a-4f9e-89d1-0118265ac11a has session id: 2d592659-e079-49e0-8976-a2d6ae10afcc","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.281652047Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Upstairs opts: Upstairs UUID: 7feeb4a4-956a-4f9e-89d1-0118265ac11a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.281691147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Crucible stats registered with UUID: 7feeb4a4-956a-4f9e-89d1-0118265ac11a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.281719177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:35.281740437Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282233182Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282257072Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520} {"msg":"7feeb4a4-956a-4f9e-89d1-0118265ac11a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282313602Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282331371Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"0","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282360661Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"0","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282375251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"1","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282395541Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"1","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282419361Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"2","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.2824402Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"2","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.28245929Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.28248085Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.28250411Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:38286)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282532869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:33474)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282560659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:36364)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.282696568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.283360972Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"0","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.283389762Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"1","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.283405352Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"2","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.283957927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"0","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284016776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"1","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284036126Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"2","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284562691Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.28467457Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[0]R flush_numbers[0..12]: [146, 144, 140, 142, 144, 145, 136, 144, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.28473256Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[0]R generation[0..12]: [81, 80, 77, 78, 80, 80, 74, 80, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284752119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284766559Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[1]R flush_numbers[0..12]: [146, 144, 140, 142, 144, 145, 136, 144, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284780149Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[1]R generation[0..12]: [81, 80, 77, 78, 80, 80, 74, 80, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284797529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284812269Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[2]R flush_numbers[0..12]: [146, 144, 140, 142, 144, 145, 136, 144, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284825729Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[2]R generation[0..12]: [81, 80, 77, 78, 80, 80, 74, 80, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284838999Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284852339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284864868Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284877748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284890408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284904338Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"":"downstairs","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284917028Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284928658Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"0","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284941738Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"1","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284954478Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"client":"2","":"downstairs","session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284967307Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.284979287Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"7feeb4a4-956a-4f9e-89d1-0118265ac11a is now active with session: 2d592659-e079-49e0-8976-a2d6ae10afcc","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.285044237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.285058826Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520,"session_id":"2d592659-e079-49e0-8976-a2d6ae10afcc"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.285071346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520} {"msg":"Activated sub_volume 7feeb4a4-956a-4f9e-89d1-0118265ac11a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:35.285082316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 37 len 16384 data: 9 6 6 7 02/30 Write block 353 len 24576 data: 4 2 3 5 7 6 03/30 Flush 04/30 Write block 307 len 32768 data: 6 7 9 7 6 6 5 5 05/30 Write block 133 len 12288 data: 5 6 6 06/30 Write block 377 len 12288 data: 10 11 10 07/30 Read block 48 len 12288 08/30 Read block 127 len 20480 09/30 Read block 83 len 16384 10/30 Read block 283 len 28672 11/30 Read block 282 len 28672 12/30 Write block 59 len 20480 data: 4 9 13 13 11 13/30 Write block 541 len 12288 data: 8 7 8 14/30 Write block 19 len 40960 data: 10 9 6 6 3 3 4 6 8 10 15/30 Read block 97 len 4096 16/30 Write block 403 len 36864 data: 11 7 4 5 4 3 3 4 6 17/30 Read block 59 len 4096 18/30 Flush 19/30 Write block 144 len 16384 data: 4 5 4 4 20/30 Read block 57 len 16384 21/30 Write block 383 len 8192 data: 7 6 22/30 Write block 290 len 36864 data: 9 9 8 7 8 7 6 5 5 23/30 Read block 11 len 32768 24/30 Read block 384 len 12288 25/30 Read block 443 len 28672 26/30 Flush 27/30 Write block 399 len 40960 data: 13 13 12 12 12 8 5 6 5 4 28/30 Write block 458 len 20480 data: 7 4 4 5 6 29/30 Read block 188 len 20480 30/30 Read block 564 len 36864 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 3 4 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:35.305341543Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 148 148 148 F F F 1 020-039 82 82 82 148 148 148 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 82 82 82 148 148 148 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 80 80 80 145 145 145 F F F 6 120-139 82 82 82 148 148 148 F F F 7 140-159 82 82 82 149 149 149 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 81 81 81 146 146 146 F F F 12 240-259 81 81 81 146 146 146 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 82 82 82 148 148 148 F F F 16 320-339 81 81 81 146 146 146 F F F 17 340-359 82 82 82 147 147 147 F F F 18 360-379 82 82 82 148 148 148 F F F 19 380-399 82 82 82 149 149 149 T T T 20 400-419 82 82 82 148 148 148 T T T 21 420-439 73 73 73 134 134 134 F F F 22 440-459 76 76 76 139 139 139 T T T 23 460-479 72 72 72 133 133 133 T T T 24 480-499 80 80 80 145 145 145 F F F 25 500-519 80 80 80 145 145 145 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 82 82 82 148 148 148 F F F 28 560-579 80 80 80 145 145 145 F F F 29 580-599 80 80 80 145 145 145 F F F Max gen: 82, Max flush: 149 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 148 148 148 F F F 1 020-039 82 82 82 148 148 148 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 82 82 82 148 148 148 F F F 4 080-099 80 80 80 144 144 144 F F F 5 100-119 80 80 80 145 145 145 F F F 6 120-139 82 82 82 148 148 148 F F F 7 140-159 82 82 82 149 149 149 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 81 81 81 146 146 146 F F F 12 240-259 81 81 81 146 146 146 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 82 82 82 148 148 148 F F F 16 320-339 81 81 81 146 146 146 F F F 17 340-359 82 82 82 147 147 147 F F F 18 360-379 82 82 82 148 148 148 F F F 19 380-399 83 83 83 150 150 150 F F F 20 400-419 83 83 83 150 150 150 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 83 83 83 150 150 150 F F F 23 460-479 83 83 83 150 150 150 F F F 24 480-499 80 80 80 145 145 145 F F F 25 500-519 80 80 80 145 145 145 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 82 82 82 148 148 148 F F F 28 560-579 80 80 80 145 145 145 F F F 29 580-599 80 80 80 145 145 145 F F F Max gen: 83, Max flush: 150 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.534625507Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.579737748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.580348973Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.580399972Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.580421972Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.580440982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528} {"msg":"Crucible 48be2fa9-fcea-44ec-8eb9-387c4398e952 has session id: 19a84be7-7fcf-4f33-ba32-1154b90d71d9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.580972727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Upstairs opts: Upstairs UUID: 48be2fa9-fcea-44ec-8eb9-387c4398e952, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581031476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Crucible stats registered with UUID: 48be2fa9-fcea-44ec-8eb9-387c4398e952","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581057596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:39.581077026Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581432263Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581457073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528} {"msg":"48be2fa9-fcea-44ec-8eb9-387c4398e952 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581516452Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581534342Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"0","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581563512Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"0","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581577502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"1","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581594822Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"1","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581623841Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"2","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581651421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"2","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.581677201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.58170133Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.58172648Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34239)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.5817513Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:60420)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.58177782Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55703)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.58180624Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.582432854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"1","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.582496203Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"0","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.582588422Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"2","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.583087498Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"1","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.583149488Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"0","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.583428925Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"2","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.583704492Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.583738592Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[0]R flush_numbers[0..12]: [148, 148, 148, 148, 144, 145, 148, 149, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584060779Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[0]R generation[0..12]: [82, 82, 82, 82, 80, 80, 82, 82, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584088999Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584104939Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[1]R flush_numbers[0..12]: [148, 148, 148, 148, 144, 145, 148, 149, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584119679Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[1]R generation[0..12]: [82, 82, 82, 82, 80, 80, 82, 82, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584133459Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584147218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[2]R flush_numbers[0..12]: [148, 148, 148, 148, 144, 145, 148, 149, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584160818Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[2]R generation[0..12]: [82, 82, 82, 82, 80, 80, 82, 82, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584174188Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584187808Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584200338Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584214098Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584226968Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584239487Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"":"downstairs","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584255467Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584275407Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"0","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584288647Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"1","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584301607Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"client":"2","":"downstairs","session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584314677Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584326617Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"48be2fa9-fcea-44ec-8eb9-387c4398e952 is now active with session: 19a84be7-7fcf-4f33-ba32-1154b90d71d9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584339187Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584350777Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528,"session_id":"19a84be7-7fcf-4f33-ba32-1154b90d71d9"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584362637Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528} {"msg":"Activated sub_volume 48be2fa9-fcea-44ec-8eb9-387c4398e952","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:39.584373616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 570 len 12288 data: 7 8 7 02/30 Read block 282 len 16384 03/30 Write block 574 len 4096 data: 6 04/30 Read block 469 len 28672 05/30 Read block 165 len 8192 06/30 Write block 95 len 24576 data: 7 5 5 7 7 7 07/30 Read block 382 len 28672 08/30 Read block 199 len 28672 09/30 Read block 230 len 24576 10/30 Read block 274 len 28672 11/30 Read block 585 len 40960 12/30 Read block 377 len 8192 13/30 Write block 350 len 40960 data: 6 5 5 5 3 4 6 8 7 7 14/30 Write block 492 len 16384 data: 6 7 7 6 15/30 Write block 579 len 8192 data: 13 11 16/30 Write block 344 len 40960 data: 5 6 5 5 5 5 7 6 6 6 17/30 Read block 240 len 4096 18/30 Read block 266 len 40960 19/30 Flush 20/30 Write block 86 len 28672 data: 9 7 9 11 10 12 11 21/30 Read block 195 len 40960 22/30 Read block 175 len 28672 23/30 Flush 24/30 Write block 545 len 28672 data: 8 8 9 10 11 9 10 25/30 Write block 507 len 32768 data: 10 9 9 8 8 8 9 8 26/30 Read block 450 len 28672 27/30 Read block 526 len 20480 28/30 Read block 316 len 40960 29/30 Read block 582 len 16384 30/30 Read block 496 len 36864 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:39.605135309Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 148 148 148 F F F 1 020-039 82 82 82 148 148 148 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 82 82 82 148 148 148 F F F 4 080-099 84 84 84 152 152 152 F F F 5 100-119 84 84 84 151 151 151 F F F 6 120-139 82 82 82 148 148 148 F F F 7 140-159 82 82 82 149 149 149 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 81 81 81 146 146 146 F F F 12 240-259 81 81 81 146 146 146 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 82 82 82 148 148 148 F F F 16 320-339 81 81 81 146 146 146 F F F 17 340-359 84 84 84 151 151 151 F F F 18 360-379 82 82 82 148 148 148 F F F 19 380-399 83 83 83 150 150 150 F F F 20 400-419 83 83 83 150 150 150 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 83 83 83 150 150 150 F F F 23 460-479 83 83 83 150 150 150 F F F 24 480-499 84 84 84 151 151 151 F F F 25 500-519 80 80 80 145 145 145 T T T 26 520-539 76 76 76 139 139 139 F F F 27 540-559 82 82 82 148 148 148 T T T 28 560-579 84 84 84 151 151 151 F F F 29 580-599 84 84 84 151 151 151 F F F Max gen: 84, Max flush: 152 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 148 148 148 F F F 1 020-039 82 82 82 148 148 148 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 82 82 82 148 148 148 F F F 4 080-099 84 84 84 152 152 152 F F F 5 100-119 84 84 84 151 151 151 F F F 6 120-139 82 82 82 148 148 148 F F F 7 140-159 82 82 82 149 149 149 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 81 81 81 146 146 146 F F F 12 240-259 81 81 81 146 146 146 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 82 82 82 148 148 148 F F F 16 320-339 81 81 81 146 146 146 F F F 17 340-359 84 84 84 151 151 151 F F F 18 360-379 82 82 82 148 148 148 F F F 19 380-399 83 83 83 150 150 150 F F F 20 400-419 83 83 83 150 150 150 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 83 83 83 150 150 150 F F F 23 460-479 83 83 83 150 150 150 F F F 24 480-499 84 84 84 151 151 151 F F F 25 500-519 85 85 85 153 153 153 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 85 85 85 153 153 153 F F F 28 560-579 84 84 84 151 151 151 F F F 29 580-599 84 84 84 151 151 151 F F F Max gen: 85, Max flush: 153 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.821972404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.865392771Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.866013796Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.866059055Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.866073745Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.866088315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536} {"msg":"Crucible 76372435-2850-4ee1-86c3-0a20624650d0 has session id: 0e60425c-f321-4e45-b8fe-b2e19ab160cc","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.86661478Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Upstairs opts: Upstairs UUID: 76372435-2850-4ee1-86c3-0a20624650d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.86664349Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Crucible stats registered with UUID: 76372435-2850-4ee1-86c3-0a20624650d0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.86665783Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:43.86667041Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867113926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867141995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536} {"msg":"76372435-2850-4ee1-86c3-0a20624650d0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867213495Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867232095Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"0","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867261274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"0","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867274954Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"1","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867288954Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"1","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867301724Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"2","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867315204Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"2","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867327834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867340944Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867354084Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34460)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867524152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:47056)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867566512Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:57373)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.867676991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.868234696Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"1","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.868276125Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"0","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.868810051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"0","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.86891619Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"1","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.869377455Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.869508514Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.870118699Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"2","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.870616874Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"2","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[0]R flush_numbers[0..12]: [148, 148, 148, 148, 152, 151, 148, 149, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871134499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[0]R generation[0..12]: [82, 82, 82, 82, 84, 84, 82, 82, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871158339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871173449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[1]R flush_numbers[0..12]: [148, 148, 148, 148, 152, 151, 148, 149, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871187609Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[1]R generation[0..12]: [82, 82, 82, 82, 84, 84, 82, 82, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871201189Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871214579Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[2]R flush_numbers[0..12]: [148, 148, 148, 148, 152, 151, 148, 149, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871227659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[2]R generation[0..12]: [82, 82, 82, 82, 84, 84, 82, 82, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871244818Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871260768Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871276348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Generation requested: 86 >= found:86"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871298628Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871314258Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871329728Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"":"downstairs","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871345777Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871360428Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"0","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871376977Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"1","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871393227Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"client":"2","":"downstairs","session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871409277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871424267Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"76372435-2850-4ee1-86c3-0a20624650d0 is now active with session: 0e60425c-f321-4e45-b8fe-b2e19ab160cc","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871439797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871454147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536,"session_id":"0e60425c-f321-4e45-b8fe-b2e19ab160cc"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871468916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536} {"msg":"Activated sub_volume 76372435-2850-4ee1-86c3-0a20624650d0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:43.871482326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 566 len 16384 data: 6 7 7 9 02/30 Read block 18 len 40960 03/30 Read block 38 len 40960 04/30 Write block 404 len 8192 data: 9 6 05/30 Write block 462 len 32768 data: 7 7 9 10 10 9 9 7 06/30 Write block 24 len 16384 data: 4 5 7 9 07/30 Read block 430 len 40960 08/30 Write block 587 len 36864 data: 8 7 5 6 6 8 10 8 8 09/30 Write block 336 len 20480 data: 10 12 9 8 8 10/30 Read block 199 len 8192 11/30 Read block 477 len 8192 12/30 Write block 591 len 24576 data: 7 9 11 9 9 7 13/30 Read block 539 len 28672 14/30 Read block 203 len 16384 15/30 Flush 16/30 Write block 345 len 36864 data: 7 6 6 6 6 8 7 7 7 17/30 Write block 451 len 12288 data: 5 7 5 18/30 Read block 170 len 28672 19/30 Read block 242 len 4096 20/30 Read block 359 len 12288 21/30 Flush 22/30 Write block 371 len 24576 data: 7 9 9 10 9 9 23/30 Flush 24/30 Write block 146 len 8192 data: 5 5 25/30 Read block 461 len 12288 26/30 Write block 76 len 36864 data: 8 9 9 8 9 10 9 9 7 27/30 Read block 124 len 36864 28/30 Read block 0 len 12288 29/30 Write block 489 len 40960 data: 8 6 6 7 8 8 7 4 5 5 30/30 Read block 376 len 28672 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:43.892322898Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 148 148 148 F F F 1 020-039 86 86 86 154 154 154 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 82 82 82 148 148 148 T T T 4 080-099 84 84 84 152 152 152 T T T 5 100-119 84 84 84 151 151 151 F F F 6 120-139 82 82 82 148 148 148 F F F 7 140-159 82 82 82 149 149 149 T T T 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 81 81 81 146 146 146 F F F 12 240-259 81 81 81 146 146 146 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 82 82 82 148 148 148 F F F 16 320-339 86 86 86 154 154 154 F F F 17 340-359 86 86 86 155 155 155 F F F 18 360-379 86 86 86 156 156 156 F F F 19 380-399 83 83 83 150 150 150 F F F 20 400-419 86 86 86 154 154 154 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 86 86 86 155 155 155 F F F 23 460-479 86 86 86 154 154 154 F F F 24 480-499 84 84 84 151 151 151 T T T 25 500-519 85 85 85 153 153 153 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 85 85 85 153 153 153 F F F 28 560-579 86 86 86 154 154 154 F F F 29 580-599 86 86 86 154 154 154 F F F Max gen: 86, Max flush: 156 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 148 148 148 F F F 1 020-039 86 86 86 154 154 154 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 87 87 87 157 157 157 F F F 4 080-099 87 87 87 157 157 157 F F F 5 100-119 84 84 84 151 151 151 F F F 6 120-139 82 82 82 148 148 148 F F F 7 140-159 87 87 87 157 157 157 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 81 81 81 146 146 146 F F F 12 240-259 81 81 81 146 146 146 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 82 82 82 148 148 148 F F F 16 320-339 86 86 86 154 154 154 F F F 17 340-359 86 86 86 155 155 155 F F F 18 360-379 86 86 86 156 156 156 F F F 19 380-399 83 83 83 150 150 150 F F F 20 400-419 86 86 86 154 154 154 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 86 86 86 155 155 155 F F F 23 460-479 86 86 86 154 154 154 F F F 24 480-499 87 87 87 157 157 157 F F F 25 500-519 85 85 85 153 153 153 F F F 26 520-539 76 76 76 139 139 139 F F F 27 540-559 85 85 85 153 153 153 F F F 28 560-579 86 86 86 154 154 154 F F F 29 580-599 86 86 86 154 154 154 F F F Max gen: 87, Max flush: 157 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.121840789Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:48.165592063Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.16597834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.197798232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.198401027Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.198443486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.198466016Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.198483776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544} {"msg":"Crucible a22d1442-93f2-4fe1-8516-80615f32d7a6 has session id: a0360e01-c192-4c99-8f86-90c2ecddfe72","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.1991692Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Upstairs opts: Upstairs UUID: a22d1442-93f2-4fe1-8516-80615f32d7a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199216689Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Crucible stats registered with UUID: a22d1442-93f2-4fe1-8516-80615f32d7a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199236929Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:48.199253549Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199693365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199717795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544} {"msg":"a22d1442-93f2-4fe1-8516-80615f32d7a6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199784224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199802324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"0","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199831964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"0","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199846354Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"1","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199860573Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"1","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199874423Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"2","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199888583Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"2","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199906303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199920803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.199933953Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"ds_connection connected from Ok(127.0.0.1:43217)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.200075942Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"ds_connection connected from Ok(127.0.0.1:47538)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.200115451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"ds_connection connected from Ok(127.0.0.1:47838)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20018243Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.200738905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"1","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.200768785Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"2","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.200890804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"0","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20139128Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"2","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.201420549Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"1","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.201597298Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"0","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202025694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202150723Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[0]R flush_numbers[0..12]: [148, 154, 148, 157, 157, 151, 148, 157, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202280712Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[0]R generation[0..12]: [82, 86, 82, 87, 87, 84, 82, 87, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202301342Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202316261Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[1]R flush_numbers[0..12]: [148, 154, 148, 157, 157, 151, 148, 157, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202330451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[1]R generation[0..12]: [82, 86, 82, 87, 87, 84, 82, 87, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202343851Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202357391Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[2]R flush_numbers[0..12]: [148, 154, 148, 157, 157, 151, 148, 157, 139, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202370661Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[2]R generation[0..12]: [82, 86, 82, 87, 87, 84, 82, 87, 76, 78, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202384051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202397501Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202410161Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20242294Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20243551Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:44:48.20244803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"":"downstairs","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20246472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20247651Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"0","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20249174Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"1","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20250476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"client":"2","":"downstairs","session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.20251752Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202529219Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"a22d1442-93f2-4fe1-8516-80615f32d7a6 is now active with session: a0360e01-c192-4c99-8f86-90c2ecddfe72","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202541659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202553359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544,"session_id":"a0360e01-c192-4c99-8f86-90c2ecddfe72"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202565299Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544} {"msg":"Activated sub_volume a22d1442-93f2-4fe1-8516-80615f32d7a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:48.202576659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 244 len 8192 data: 5 5 02/30 Read block 18 len 8192 03/30 Write block 563 len 20480 data: 4 5 8 7 8 04/30 Read block 423 len 24576 05/30 Write block 409 len 20480 data: 4 5 7 7 7 06/30 Write block 523 len 20480 data: 6 8 8 9 9 07/30 Read block 585 len 12288 08/30 Read block 464 len 20480 09/30 Write block 302 len 12288 data: 7 7 7 10/30 Flush 11/30 Flush 12/30 Write block 15 len 24576 data: 8 9 10 10 11 10 13/30 Write block 215 len 12288 data: 6 8 9 14/30 Read block 182 len 32768 15/30 Write block 150 len 28672 data: 5 6 7 6 7 9 8 16/30 Write block 228 len 32768 data: 7 9 8 7 6 9 11 12 17/30 Read block 138 len 16384 18/30 Read block 580 len 12288 19/30 Flush 20/30 Write block 386 len 12288 data: 7 8 12 21/30 Flush 22/30 Write block 108 len 28672 data: 8 7 9 9 7 8 6 23/30 Read block 1 len 16384 24/30 Write block 239 len 20480 data: 8 11 8 7 6 25/30 Read block 267 len 16384 26/30 Read block 476 len 24576 27/30 Write block 514 len 20480 data: 9 8 10 12 8 28/30 Read block 83 len 4096 29/30 Read block 20 len 36864 30/30 Read block 527 len 20480 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 7 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:48.222718037Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 160 160 160 F F F 1 020-039 88 88 88 160 160 160 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 87 87 87 157 157 157 F F F 4 080-099 87 87 87 157 157 157 F F F 5 100-119 84 84 84 151 151 151 T T T 6 120-139 82 82 82 148 148 148 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 88 88 88 160 160 160 F F F 11 220-239 88 88 88 160 160 160 T T T 12 240-259 88 88 88 158 158 158 T T T 13 260-279 80 80 80 145 145 145 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 86 86 86 154 154 154 F F F 17 340-359 86 86 86 155 155 155 F F F 18 360-379 86 86 86 156 156 156 F F F 19 380-399 88 88 88 161 161 161 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 86 86 86 155 155 155 F F F 23 460-479 86 86 86 154 154 154 F F F 24 480-499 87 87 87 157 157 157 F F F 25 500-519 85 85 85 153 153 153 T T T 26 520-539 88 88 88 158 158 158 F F F 27 540-559 85 85 85 153 153 153 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 86 86 86 154 154 154 F F F Max gen: 88, Max flush: 161 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 160 160 160 F F F 1 020-039 88 88 88 160 160 160 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 87 87 87 157 157 157 F F F 4 080-099 87 87 87 157 157 157 F F F 5 100-119 89 89 89 162 162 162 F F F 6 120-139 82 82 82 148 148 148 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 88 88 88 160 160 160 F F F 11 220-239 89 89 89 162 162 162 F F F 12 240-259 89 89 89 162 162 162 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 82 82 82 149 149 149 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 86 86 86 154 154 154 F F F 17 340-359 86 86 86 155 155 155 F F F 18 360-379 86 86 86 156 156 156 F F F 19 380-399 88 88 88 161 161 161 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 73 73 73 134 134 134 F F F 22 440-459 86 86 86 155 155 155 F F F 23 460-479 86 86 86 154 154 154 F F F 24 480-499 87 87 87 157 157 157 F F F 25 500-519 89 89 89 162 162 162 F F F 26 520-539 88 88 88 158 158 158 F F F 27 540-559 85 85 85 153 153 153 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 86 86 86 154 154 154 F F F Max gen: 89, Max flush: 162 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.48517277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.529052843Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.529747197Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.529789136Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.529803516Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.529817696Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552} {"msg":"Crucible 0324dea5-c97f-48cb-b0dd-19548d49d624 has session id: 0d0c8140-c022-4301-9527-f46987859a0a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.530340851Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Upstairs opts: Upstairs UUID: 0324dea5-c97f-48cb-b0dd-19548d49d624, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.530381671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Crucible stats registered with UUID: 0324dea5-c97f-48cb-b0dd-19548d49d624","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.530407421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:52.530429531Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.530877817Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.530899756Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552} {"msg":"0324dea5-c97f-48cb-b0dd-19548d49d624 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.530964746Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.530986116Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"0","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531031105Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"0","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531046255Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"1","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531060805Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"1","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531073895Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"2","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531087585Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"2","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531100435Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531125194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531149434Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56697)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531172754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33183)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531195634Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"ds_connection connected from Ok(127.0.0.1:60286)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.531320473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.532027846Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"0","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.532056686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"1","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.532555211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"2","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.53264703Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"1","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.53273177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"0","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533235095Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"2","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533392734Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533428474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[0]R flush_numbers[0..12]: [160, 160, 148, 157, 157, 162, 148, 160, 139, 142, 160, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533709671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[0]R generation[0..12]: [88, 88, 82, 87, 87, 89, 82, 88, 76, 78, 88, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533732241Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533747951Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[1]R flush_numbers[0..12]: [160, 160, 148, 157, 157, 162, 148, 160, 139, 142, 160, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.53376156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[1]R generation[0..12]: [88, 88, 82, 87, 87, 89, 82, 88, 76, 78, 88, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.53377569Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.53378986Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[2]R flush_numbers[0..12]: [160, 160, 148, 157, 157, 162, 148, 160, 139, 142, 160, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.53380338Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[2]R generation[0..12]: [88, 88, 82, 87, 87, 89, 82, 88, 76, 78, 88, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.53381647Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.53383408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T01:44:52.533859899Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533942049Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533957679Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"":"downstairs","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533970578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533982298Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"0","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.533995198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"1","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.534007888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"client":"2","":"downstairs","session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.534020638Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.534032578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"0324dea5-c97f-48cb-b0dd-19548d49d624 is now active with session: 0d0c8140-c022-4301-9527-f46987859a0a","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.534045138Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.534056818Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552,"session_id":"0d0c8140-c022-4301-9527-f46987859a0a"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.534068788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552} {"msg":"Activated sub_volume 0324dea5-c97f-48cb-b0dd-19548d49d624","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:52.534079977Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 340 len 4096 data: 9 02/30 Write block 595 len 8192 data: 10 8 03/30 Read block 252 len 40960 04/30 Flush 05/30 Write block 350 len 24576 data: 9 8 8 8 4 5 06/30 Read block 186 len 40960 07/30 Write block 81 len 24576 data: 11 10 10 8 7 10 08/30 Read block 455 len 4096 09/30 Read block 194 len 12288 10/30 Write block 285 len 20480 data: 6 7 7 7 8 11/30 Flush 12/30 Write block 341 len 4096 data: 8 13/30 Write block 220 len 16384 data: 10 13 12 11 14/30 Write block 12 len 32768 data: 9 8 6 9 10 11 11 12 15/30 Read block 342 len 36864 16/30 Read block 40 len 36864 17/30 Read block 177 len 36864 18/30 Read block 521 len 4096 19/30 Write block 26 len 20480 data: 8 10 11 8 8 20/30 Read block 287 len 36864 21/30 Write block 119 len 40960 data: 5 9 10 10 11 9 7 7 7 4 22/30 Read block 322 len 36864 23/30 Write block 426 len 24576 data: 8 6 8 8 8 7 24/30 Read block 206 len 16384 25/30 Write block 355 len 40960 data: 6 7 9 8 8 8 6 5 5 5 26/30 Flush 27/30 Write block 541 len 20480 data: 9 8 9 7 9 28/30 Write block 583 len 16384 data: 8 9 7 6 29/30 Read block 45 len 36864 30/30 Write block 431 len 40960 data: 8 7 5 5 6 5 5 5 4 4 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:52.554657642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 165 165 165 F F F 1 020-039 90 90 90 165 165 165 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 87 87 87 157 157 157 F F F 4 080-099 90 90 90 164 164 164 F F F 5 100-119 90 90 90 165 165 165 F F F 6 120-139 90 90 90 165 165 165 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 88 88 88 160 160 160 F F F 11 220-239 90 90 90 165 165 165 F F F 12 240-259 89 89 89 162 162 162 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 90 90 90 164 164 164 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 86 86 86 154 154 154 F F F 17 340-359 90 90 90 165 165 165 F F F 18 360-379 90 90 90 165 165 165 F F F 19 380-399 88 88 88 161 161 161 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 90 90 90 165 165 165 T T T 22 440-459 86 86 86 155 155 155 T T T 23 460-479 86 86 86 154 154 154 F F F 24 480-499 87 87 87 157 157 157 F F F 25 500-519 89 89 89 162 162 162 F F F 26 520-539 88 88 88 158 158 158 F F F 27 540-559 85 85 85 153 153 153 T T T 28 560-579 88 88 88 158 158 158 F F F 29 580-599 90 90 90 163 163 163 T T T Max gen: 90, Max flush: 165 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 165 165 165 F F F 1 020-039 90 90 90 165 165 165 F F F 2 040-059 82 82 82 148 148 148 F F F 3 060-079 87 87 87 157 157 157 F F F 4 080-099 90 90 90 164 164 164 F F F 5 100-119 90 90 90 165 165 165 F F F 6 120-139 90 90 90 165 165 165 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 88 88 88 160 160 160 F F F 11 220-239 90 90 90 165 165 165 F F F 12 240-259 89 89 89 162 162 162 F F F 13 260-279 80 80 80 145 145 145 F F F 14 280-299 90 90 90 164 164 164 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 86 86 86 154 154 154 F F F 17 340-359 90 90 90 165 165 165 F F F 18 360-379 90 90 90 165 165 165 F F F 19 380-399 88 88 88 161 161 161 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 91 91 91 166 166 166 F F F 22 440-459 91 91 91 166 166 166 F F F 23 460-479 86 86 86 154 154 154 F F F 24 480-499 87 87 87 157 157 157 F F F 25 500-519 89 89 89 162 162 162 F F F 26 520-539 88 88 88 158 158 158 F F F 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 91 91 91 166 166 166 F F F Max gen: 91, Max flush: 166 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.788999369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:56.831789392Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.832149148Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.863152348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.863697563Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.863731163Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.863744893Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.863759442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560} {"msg":"Crucible b6d5d577-137d-42a3-87be-c02194ff54dd has session id: 45f5b69b-610d-4e02-a429-95ed9b282466","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864291508Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Upstairs opts: Upstairs UUID: b6d5d577-137d-42a3-87be-c02194ff54dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864330697Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Crucible stats registered with UUID: b6d5d577-137d-42a3-87be-c02194ff54dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864353647Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:56.864373357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864813303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864840563Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560} {"msg":"b6d5d577-137d-42a3-87be-c02194ff54dd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864897672Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864915762Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"0","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864952311Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"0","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.864977611Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"1","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.865006311Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"1","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.865034051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"2","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.865060101Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"2","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.86508582Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.86510325Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.86513184Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"ds_connection connected from Ok(127.0.0.1:35501)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.86516076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"ds_connection connected from Ok(127.0.0.1:38461)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.865188369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"ds_connection connected from Ok(127.0.0.1:57953)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.865216499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.865932233Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"1","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.865961073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"2","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.865976932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"0","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.866587757Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"1","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.866787905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"0","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.866832145Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"2","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867227171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.86733829Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[0]R flush_numbers[0..12]: [165, 165, 148, 157, 164, 165, 165, 160, 139, 142, 160, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867460479Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[0]R generation[0..12]: [90, 90, 82, 87, 90, 90, 90, 88, 76, 78, 88, 90]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867480849Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867498699Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[1]R flush_numbers[0..12]: [165, 165, 148, 157, 164, 165, 165, 160, 139, 142, 160, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867513128Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[1]R generation[0..12]: [90, 90, 82, 87, 90, 90, 90, 88, 76, 78, 88, 90]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867526728Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867540318Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[2]R flush_numbers[0..12]: [165, 165, 148, 157, 164, 165, 165, 160, 139, 142, 160, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867553528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[2]R generation[0..12]: [90, 90, 82, 87, 90, 90, 90, 88, 76, 78, 88, 90]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867566528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867588708Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867613078Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867637497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867656767Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867677107Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"":"downstairs","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:44:56.867701757Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867725926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"0","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867743226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"1","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867764406Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"client":"2","":"downstairs","session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867788956Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867812336Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"b6d5d577-137d-42a3-87be-c02194ff54dd is now active with session: 45f5b69b-610d-4e02-a429-95ed9b282466","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867835605Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867857145Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560,"session_id":"45f5b69b-610d-4e02-a429-95ed9b282466"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867876015Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560} {"msg":"Activated sub_volume b6d5d577-137d-42a3-87be-c02194ff54dd","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:44:56.867887455Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 22 len 24576 data: 7 4 5 6 9 11 02/30 Read block 53 len 8192 03/30 Read block 50 len 4096 04/30 Write block 468 len 4096 data: 10 05/30 Write block 48 len 36864 data: 6 6 7 8 10 7 9 10 11 06/30 Read block 512 len 24576 07/30 Write block 439 len 40960 data: 5 5 3 5 5 9 7 7 8 8 08/30 Read block 280 len 28672 09/30 Read block 408 len 28672 10/30 Flush 11/30 Write block 439 len 36864 data: 6 6 4 6 6 10 8 8 9 12/30 Flush 13/30 Write block 216 len 20480 data: 9 10 10 10 11 14/30 Write block 448 len 32768 data: 9 7 5 6 8 6 6 7 15/30 Write block 78 len 12288 data: 10 9 10 16/30 Read block 571 len 32768 17/30 Read block 226 len 4096 18/30 Read block 536 len 16384 19/30 Flush 20/30 Write block 5 len 20480 data: 6 8 11 11 9 21/30 Write block 260 len 24576 data: 6 4 5 5 8 8 22/30 Read block 200 len 8192 23/30 Write block 513 len 40960 data: 10 10 9 11 13 9 10 9 11 8 24/30 Write block 106 len 20480 data: 9 8 9 8 10 25/30 Read block 193 len 24576 26/30 Write block 279 len 24576 data: 10 10 10 13 11 10 27/30 Write block 445 len 36864 data: 9 9 10 10 8 6 7 9 7 28/30 Write block 327 len 8192 data: 9 10 29/30 Read block 572 len 8192 30/30 Read block 347 len 28672 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 11 11 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:44:56.888116112Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 165 165 165 T T T 1 020-039 92 92 92 167 167 167 F F F 2 040-059 92 92 92 167 167 167 F F F 3 060-079 92 92 92 169 169 169 F F F 4 080-099 92 92 92 169 169 169 F F F 5 100-119 90 90 90 165 165 165 T T T 6 120-139 90 90 90 165 165 165 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 92 92 92 169 169 169 F F F 11 220-239 92 92 92 169 169 169 F F F 12 240-259 89 89 89 162 162 162 F F F 13 260-279 80 80 80 145 145 145 T T T 14 280-299 90 90 90 164 164 164 T T T 15 300-319 88 88 88 158 158 158 F F F 16 320-339 86 86 86 154 154 154 T T T 17 340-359 90 90 90 165 165 165 F F F 18 360-379 90 90 90 165 165 165 F F F 19 380-399 88 88 88 161 161 161 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 92 92 92 168 168 168 F F F 22 440-459 92 92 92 169 169 169 T T T 23 460-479 92 92 92 167 167 167 F F F 24 480-499 87 87 87 157 157 157 F F F 25 500-519 89 89 89 162 162 162 T T T 26 520-539 88 88 88 158 158 158 T T T 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 91 91 91 166 166 166 F F F Max gen: 92, Max flush: 169 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 170 170 170 F F F 1 020-039 92 92 92 167 167 167 F F F 2 040-059 92 92 92 167 167 167 F F F 3 060-079 92 92 92 169 169 169 F F F 4 080-099 92 92 92 169 169 169 F F F 5 100-119 93 93 93 170 170 170 F F F 6 120-139 90 90 90 165 165 165 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 92 92 92 169 169 169 F F F 11 220-239 92 92 92 169 169 169 F F F 12 240-259 89 89 89 162 162 162 F F F 13 260-279 93 93 93 170 170 170 F F F 14 280-299 93 93 93 170 170 170 F F F 15 300-319 88 88 88 158 158 158 F F F 16 320-339 93 93 93 170 170 170 F F F 17 340-359 90 90 90 165 165 165 F F F 18 360-379 90 90 90 165 165 165 F F F 19 380-399 88 88 88 161 161 161 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 92 92 92 168 168 168 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 92 92 92 167 167 167 F F F 24 480-499 87 87 87 157 157 157 F F F 25 500-519 93 93 93 170 170 170 F F F 26 520-539 93 93 93 170 170 170 F F F 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 91 91 91 166 166 166 F F F Max gen: 93, Max flush: 170 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.174826476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:01.219206314Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.219592501Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.250830868Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.251535312Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.251571192Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.251585031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.251598901Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568} {"msg":"Crucible 4c0e82fc-a154-4fca-9400-e61ccf9edfed has session id: 4e785e2f-ae25-4966-a253-f403a404bd03","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.252300305Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Upstairs opts: Upstairs UUID: 4c0e82fc-a154-4fca-9400-e61ccf9edfed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.252330594Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Crucible stats registered with UUID: 4c0e82fc-a154-4fca-9400-e61ccf9edfed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.252346645Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:01.252359294Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.25278471Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.25280941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568} {"msg":"4c0e82fc-a154-4fca-9400-e61ccf9edfed active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.25287714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.2528952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"0","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.252925119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"0","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.252939249Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"1","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.252953539Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"1","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.252966519Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"2","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.253019618Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"2","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.253056858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.253085128Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.253110458Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"ds_connection connected from Ok(127.0.0.1:45752)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.253136717Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"ds_connection connected from Ok(127.0.0.1:50222)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.253163647Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"ds_connection connected from Ok(127.0.0.1:34392)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.253280396Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.25397114Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"2","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.25400017Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"0","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.254021529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"1","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.254628744Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"2","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.254666324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"1","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.254800282Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"0","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255228878Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255335857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[0]R flush_numbers[0..12]: [170, 167, 167, 169, 169, 170, 165, 160, 139, 142, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255417797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[0]R generation[0..12]: [93, 92, 92, 92, 92, 93, 90, 88, 76, 78, 92, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255438037Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255452656Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[1]R flush_numbers[0..12]: [170, 167, 167, 169, 169, 170, 165, 160, 139, 142, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255466346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[1]R generation[0..12]: [93, 92, 92, 92, 92, 93, 90, 88, 76, 78, 92, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255479606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255493346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[2]R flush_numbers[0..12]: [170, 167, 167, 169, 169, 170, 165, 160, 139, 142, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255507016Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[2]R generation[0..12]: [93, 92, 92, 92, 92, 93, 90, 88, 76, 78, 92, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255520226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255537416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255550376Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255563245Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255581545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255597445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"":"downstairs","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255610625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255622405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"0","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255635465Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"1","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255655615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"client":"2","":"downstairs","session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255668574Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255680444Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"4c0e82fc-a154-4fca-9400-e61ccf9edfed is now active with session: 4e785e2f-ae25-4966-a253-f403a404bd03","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255692904Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255704434Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568,"session_id":"4e785e2f-ae25-4966-a253-f403a404bd03"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255716464Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568} {"msg":"Activated sub_volume 4c0e82fc-a154-4fca-9400-e61ccf9edfed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:01.255727364Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 100 len 32768 data: 8 9 9 9 5 8 10 9 02/30 Read block 87 len 16384 03/30 Read block 441 len 24576 04/30 Write block 318 len 4096 data: 8 05/30 Write block 240 len 40960 data: 12 9 8 7 6 6 5 4 5 5 06/30 Write block 469 len 28672 data: 8 8 9 6 5 5 5 07/30 Read block 63 len 8192 08/30 Read block 565 len 24576 09/30 Flush 10/30 Write block 593 len 20480 data: 12 10 11 9 4 11/30 Read block 281 len 40960 12/30 Write block 435 len 12288 data: 7 6 6 13/30 Read block 505 len 20480 14/30 Read block 116 len 32768 15/30 Read block 355 len 4096 16/30 Read block 83 len 20480 17/30 Flush 18/30 Write block 387 len 24576 data: 9 13 13 14 13 14 19/30 Read block 589 len 32768 20/30 Read block 514 len 40960 21/30 Read block 488 len 16384 22/30 Read block 23 len 12288 23/30 Write block 351 len 4096 data: 9 24/30 Write block 489 len 4096 data: 9 25/30 Read block 583 len 24576 26/30 Read block 133 len 36864 27/30 Read block 295 len 40960 28/30 Write block 270 len 36864 data: 8 9 8 9 8 7 6 8 10 29/30 Read block 157 len 40960 30/30 Read block 492 len 20480 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:01.276897082Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 170 170 170 F F F 1 020-039 92 92 92 167 167 167 F F F 2 040-059 92 92 92 167 167 167 F F F 3 060-079 92 92 92 169 169 169 F F F 4 080-099 92 92 92 169 169 169 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 90 90 90 165 165 165 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 92 92 92 169 169 169 F F F 11 220-239 92 92 92 169 169 169 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 93 93 93 170 170 170 T T T 14 280-299 93 93 93 170 170 170 F F F 15 300-319 94 94 94 171 171 171 F F F 16 320-339 93 93 93 170 170 170 F F F 17 340-359 90 90 90 165 165 165 T T T 18 360-379 90 90 90 165 165 165 F F F 19 380-399 88 88 88 161 161 161 T T T 20 400-419 88 88 88 158 158 158 F F F 21 420-439 94 94 94 172 172 172 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 94 94 94 171 171 171 F F F 24 480-499 87 87 87 157 157 157 T T T 25 500-519 93 93 93 170 170 170 F F F 26 520-539 93 93 93 170 170 170 F F F 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 94 94 94 172 172 172 F F F Max gen: 94, Max flush: 172 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 170 170 170 F F F 1 020-039 92 92 92 167 167 167 F F F 2 040-059 92 92 92 167 167 167 F F F 3 060-079 92 92 92 169 169 169 F F F 4 080-099 92 92 92 169 169 169 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 90 90 90 165 165 165 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 78 78 78 142 142 142 F F F 10 200-219 92 92 92 169 169 169 F F F 11 220-239 92 92 92 169 169 169 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 95 95 95 173 173 173 F F F 14 280-299 93 93 93 170 170 170 F F F 15 300-319 94 94 94 171 171 171 F F F 16 320-339 93 93 93 170 170 170 F F F 17 340-359 95 95 95 173 173 173 F F F 18 360-379 90 90 90 165 165 165 F F F 19 380-399 95 95 95 173 173 173 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 94 94 94 172 172 172 F F F 22 440-459 93 93 93 170 170 170 F F F 23 460-479 94 94 94 171 171 171 F F F 24 480-499 95 95 95 173 173 173 F F F 25 500-519 93 93 93 170 170 170 F F F 26 520-539 93 93 93 170 170 170 F F F 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 94 94 94 172 172 172 F F F Max gen: 95, Max flush: 173 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.540894052Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:05.583589716Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.584017222Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.614702344Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.615252549Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.615291519Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.615309218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.615324518Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576} {"msg":"Crucible 65cca825-04fb-4b59-ba6f-34483ad50d26 has session id: 569e6a6a-983b-4e86-8437-82503025e586","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.615785464Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Upstairs opts: Upstairs UUID: 65cca825-04fb-4b59-ba6f-34483ad50d26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.615888043Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Crucible stats registered with UUID: 65cca825-04fb-4b59-ba6f-34483ad50d26","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.615917043Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:05.615939773Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616382369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616409409Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576} {"msg":"65cca825-04fb-4b59-ba6f-34483ad50d26 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616471388Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616489888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"0","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616519618Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"0","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616533667Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"1","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616557657Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"1","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616585067Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"2","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616614957Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"2","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616638806Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616662436Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616687176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"ds_connection connected from Ok(127.0.0.1:47730)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616712696Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"ds_connection connected from Ok(127.0.0.1:61551)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616738316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"ds_connection connected from Ok(127.0.0.1:53577)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.616765815Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.6173526Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"2","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.61739127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"1","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.617440699Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"0","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.617958005Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"1","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.617988994Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"0","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618138723Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"2","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618610369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618637419Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[0]R flush_numbers[0..12]: [170, 167, 167, 169, 169, 171, 165, 160, 139, 142, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618750898Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[0]R generation[0..12]: [93, 92, 92, 92, 92, 94, 90, 88, 76, 78, 92, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618772287Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618786997Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[1]R flush_numbers[0..12]: [170, 167, 167, 169, 169, 171, 165, 160, 139, 142, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618800857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[1]R generation[0..12]: [93, 92, 92, 92, 92, 94, 90, 88, 76, 78, 92, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618814467Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618828557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[2]R flush_numbers[0..12]: [170, 167, 167, 169, 169, 171, 165, 160, 139, 142, 169, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618842967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[2]R generation[0..12]: [93, 92, 92, 92, 92, 94, 90, 88, 76, 78, 92, 92]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618856526Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618870816Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618883476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618896266Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618908796Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618921476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"":"downstairs","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618934316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618949186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"0","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618962336Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"client":"1","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618975076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T"Wait for a query_work_queue command to finish before sending IO ,"pid":1576,"client":"2","":"downstairs","session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.618995025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.619007225Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"65cca825-04fb-4b59-ba6f-34483ad50d26 is now active with session: 569e6a6a-983b-4e86-8437-82503025e586","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.619019845Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.619032165Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576,"session_id":"569e6a6a-983b-4e86-8437-82503025e586"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.619044215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576} {"msg":"Activated sub_volume 65cca825-04fb-4b59-ba6f-34483ad50d26","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:05.619055155Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 493 len 40960 data: 9 9 8 5 6 6 5 6 7 6 02/30 Write block 20 len 40960 data: 11 7 8 5 6 7 10 12 12 9 03/30 Write block 128 len 20480 data: 5 4 4 4 3 04/30 Read block 363 len 24576 05/30 Read block 451 len 20480 06/30 Write block 273 len 36864 data: 10 9 8 7 9 11 11 11 11 07/30 Flush 08/30 Write block 316 len 8192 data: 8 6 09/30 Read block 64 len 4096 10/30 Read block 64 len 20480 11/30 Read block 552 len 28672 12/30 Read block 538 len 40960 13/30 Read block 159 len 28672 14/30 Write block 212 len 36864 data: 8 8 9 7 10 11 11 11 12 15/30 Read block 586 len 4096 16/30 Read block 261 len 16384 17/30 Write block 120 len 12288 data: 10 11 11 18/30 Write block 75 len 36864 data: 6 9 10 11 10 11 12 11 11 19/30 Read block 377 len 40960 20/30 Write block 390 len 20480 data: 15 14 15 13 12 21/30 Write block 182 len 4096 data: 6 22/30 Write block 449 len 8192 data: 9 7 23/30 Read block 381 len 8192 24/30 Flush 25/30 Write block 317 len 24576 data: 7 9 8 8 9 7 26/30 Read block 583 len 12288 27/30 Read block 10 len 16384 28/30 Write block 286 len 40960 data: 8 8 8 9 10 10 9 8 9 8 29/30 Read block 436 len 8192 30/30 Write block 530 len 12288 data: 8 5 6 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:05.640194813Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 170 170 170 F F F 1 020-039 96 96 96 174 174 174 F F F 2 040-059 92 92 92 167 167 167 F F F 3 060-079 96 96 96 175 175 175 F F F 4 080-099 96 96 96 175 175 175 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 96 96 96 175 175 175 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 96 96 96 175 175 175 F F F 10 200-219 96 96 96 175 175 175 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 96 96 96 174 174 174 F F F 14 280-299 96 96 96 174 174 174 T T T 15 300-319 96 96 96 175 175 175 T T T 16 320-339 93 93 93 170 170 170 T T T 17 340-359 95 95 95 173 173 173 F F F 18 360-379 90 90 90 165 165 165 F F F 19 380-399 96 96 96 175 175 175 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 94 94 94 172 172 172 F F F 22 440-459 96 96 96 175 175 175 F F F 23 460-479 94 94 94 171 171 171 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 96 96 96 174 174 174 F F F 26 520-539 93 93 93 170 170 170 T T T 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 94 94 94 172 172 172 F F F Max gen: 96, Max flush: 175 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 170 170 170 F F F 1 020-039 96 96 96 174 174 174 F F F 2 040-059 92 92 92 167 167 167 F F F 3 060-079 96 96 96 175 175 175 F F F 4 080-099 96 96 96 175 175 175 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 96 96 96 175 175 175 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 76 76 76 139 139 139 F F F 9 180-199 96 96 96 175 175 175 F F F 10 200-219 96 96 96 175 175 175 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 96 96 96 174 174 174 F F F 14 280-299 97 97 97 176 176 176 F F F 15 300-319 97 97 97 176 176 176 F F F 16 320-339 97 97 97 176 176 176 F F F 17 340-359 95 95 95 173 173 173 F F F 18 360-379 90 90 90 165 165 165 F F F 19 380-399 96 96 96 175 175 175 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 94 94 94 172 172 172 F F F 22 440-459 96 96 96 175 175 175 F F F 23 460-479 94 94 94 171 171 171 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 96 96 96 174 174 174 F F F 26 520-539 97 97 97 176 176 176 F F F 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 94 94 94 172 172 172 F F F Max gen: 97, Max flush: 176 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.903696467Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.947372502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.947955967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.947999036Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.948014106Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.948029496Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584} {"msg":"Crucible bc4cc632-b2fb-4e33-9fa9-4a78a04c1746 has session id: ab7a6914-5675-4f32-964c-ccd3b417cf13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.948621241Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Upstairs opts: Upstairs UUID: bc4cc632-b2fb-4e33-9fa9-4a78a04c1746, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.94867245Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Crucible stats registered with UUID: bc4cc632-b2fb-4e33-9fa9-4a78a04c1746","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.94870357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:09.94872865Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949124976Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949152296Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584} {"msg":"bc4cc632-b2fb-4e33-9fa9-4a78a04c1746 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949212845Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949236915Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"0","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949268795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"0","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949293135Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"1","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949320824Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"1","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949349404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"2","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949381244Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"2","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949410303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949437553Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.949463463Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"ds_connection connected from Ok(127.0.0.1:55335)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.94977144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"ds_connection connected from Ok(127.0.0.1:33670)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.94980298Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"ds_connection connected from Ok(127.0.0.1:46576)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.94981869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.950525203Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"0","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.950553723Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"1","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.950573273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"2","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951168958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"0","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951197237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"1","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951222187Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"2","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951799432Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951834212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[0]R flush_numbers[0..12]: [170, 174, 167, 175, 175, 171, 175, 160, 139, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951853421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[0]R generation[0..12]: [93, 96, 92, 96, 96, 94, 96, 88, 76, 96, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951868451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951882941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[1]R flush_numbers[0..12]: [170, 174, 167, 175, 175, 171, 175, 160, 139, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951896541Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[1]R generation[0..12]: [93, 96, 92, 96, 96, 94, 96, 88, 76, 96, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951909741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951923351Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[2]R flush_numbers[0..12]: [170, 174, 167, 175, 175, 171, 175, 160, 139, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.951936591Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[2]R generation[0..12]: [93, 96, 92, 96, 96, 94, 96, 88, 76, 96, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.95194958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.95196692Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.95198616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.95200991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.95203175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.95205419Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"":"downstairs","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952069299Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:45:09.952081419Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"0","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952098779Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"1","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952111529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"client":"2","":"downstairs","session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952124249Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952136159Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"bc4cc632-b2fb-4e33-9fa9-4a78a04c1746 is now active with session: ab7a6914-5675-4f32-964c-ccd3b417cf13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952148489Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952160059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584,"session_id":"ab7a6914-5675-4f32-964c-ccd3b417cf13"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952171918Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584} {"msg":"Activated sub_volume bc4cc632-b2fb-4e33-9fa9-4a78a04c1746","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:09.952182848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 340 len 8192 data: 10 9 02/30 Read block 566 len 8192 03/30 Read block 403 len 4096 04/30 Write block 172 len 32768 data: 3 5 5 7 5 5 5 6 05/30 Write block 128 len 8192 data: 6 5 06/30 Read block 31 len 16384 07/30 Flush 08/30 Write block 374 len 8192 data: 11 10 09/30 Read block 288 len 4096 10/30 Read block 125 len 24576 11/30 Read block 138 len 36864 12/30 Read block 314 len 32768 13/30 Write block 467 len 20480 data: 10 11 9 9 10 14/30 Read block 453 len 20480 15/30 Read block 128 len 12288 16/30 Read block 532 len 36864 17/30 Read block 576 len 8192 18/30 Write block 300 len 16384 data: 6 6 8 8 19/30 Write block 1 len 12288 data: 4 4 4 20/30 Read block 535 len 40960 21/30 Read block 11 len 8192 22/30 Read block 196 len 8192 23/30 Read block 443 len 24576 24/30 Read block 129 len 8192 25/30 Read block 429 len 20480 26/30 Write block 309 len 28672 data: 10 8 7 7 6 6 5 27/30 Read block 257 len 24576 28/30 Read block 218 len 12288 29/30 Write block 153 len 28672 data: 7 8 10 9 5 5 5 30/30 Read block 415 len 20480 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 5 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 23 22 21 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:09.972173077Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 170 170 170 T T T 1 020-039 96 96 96 174 174 174 F F F 2 040-059 92 92 92 167 167 167 F F F 3 060-079 96 96 96 175 175 175 F F F 4 080-099 96 96 96 175 175 175 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 98 98 98 177 177 177 F F F 7 140-159 88 88 88 160 160 160 T T T 8 160-179 98 98 98 177 177 177 F F F 9 180-199 96 96 96 175 175 175 F F F 10 200-219 96 96 96 175 175 175 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 96 96 96 174 174 174 F F F 14 280-299 97 97 97 176 176 176 F F F 15 300-319 97 97 97 176 176 176 T T T 16 320-339 97 97 97 176 176 176 F F F 17 340-359 98 98 98 177 177 177 F F F 18 360-379 90 90 90 165 165 165 T T T 19 380-399 96 96 96 175 175 175 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 94 94 94 172 172 172 F F F 22 440-459 96 96 96 175 175 175 F F F 23 460-479 94 94 94 171 171 171 T T T 24 480-499 96 96 96 174 174 174 F F F 25 500-519 96 96 96 174 174 174 F F F 26 520-539 97 97 97 176 176 176 F F F 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 94 94 94 172 172 172 F F F Max gen: 98, Max flush: 177 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 178 178 178 F F F 1 020-039 96 96 96 174 174 174 F F F 2 040-059 92 92 92 167 167 167 F F F 3 060-079 96 96 96 175 175 175 F F F 4 080-099 96 96 96 175 175 175 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 98 98 98 177 177 177 F F F 7 140-159 99 99 99 178 178 178 F F F 8 160-179 98 98 98 177 177 177 F F F 9 180-199 96 96 96 175 175 175 F F F 10 200-219 96 96 96 175 175 175 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 96 96 96 174 174 174 F F F 14 280-299 97 97 97 176 176 176 F F F 15 300-319 99 99 99 178 178 178 F F F 16 320-339 97 97 97 176 176 176 F F F 17 340-359 98 98 98 177 177 177 F F F 18 360-379 99 99 99 178 178 178 F F F 19 380-399 96 96 96 175 175 175 F F F 20 400-419 88 88 88 158 158 158 F F F 21 420-439 94 94 94 172 172 172 F F F 22 440-459 96 96 96 175 175 175 F F F 23 460-479 99 99 99 178 178 178 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 96 96 96 174 174 174 F F F 26 520-539 97 97 97 176 176 176 F F F 27 540-559 91 91 91 166 166 166 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 94 94 94 172 172 172 F F F Max gen: 99, Max flush: 178 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.219157351Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:24.264296793Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.264660529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.296901147Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.29768274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.29772606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.29776708Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.297787059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592} {"msg":"Crucible a4b30d20-fa7f-41af-a674-cc0ae0acb02f has session id: 3ae0d177-e291-4978-aec7-493e503df98e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.298429804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Upstairs opts: Upstairs UUID: a4b30d20-fa7f-41af-a674-cc0ae0acb02f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.298460283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Crucible stats registered with UUID: a4b30d20-fa7f-41af-a674-cc0ae0acb02f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.298475963Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:24.298489363Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.29886856Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.2988926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592} {"msg":"a4b30d20-fa7f-41af-a674-cc0ae0acb02f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.298954169Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.298971339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"0","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.298999668Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"0","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299013408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"1","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299027968Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"1","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299044808Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"2","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299059258Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"2","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299072198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299085868Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299099018Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61713)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299200297Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34933)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299243026Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40461)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.299273426Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.29994306Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"1","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.2999751Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"2","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.300182888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"0","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.300642654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"2","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.300682203Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"1","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.300778263Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"0","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301329578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301366017Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[0]R flush_numbers[0..12]: [178, 174, 167, 175, 175, 171, 177, 178, 177, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301464896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[0]R generation[0..12]: [99, 96, 92, 96, 96, 94, 98, 99, 98, 96, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301493486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301509306Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[1]R flush_numbers[0..12]: [178, 174, 167, 175, 175, 171, 177, 178, 177, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301523466Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[1]R generation[0..12]: [99, 96, 92, 96, 96, 94, 98, 99, 98, 96, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301537156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301551076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[2]R flush_numbers[0..12]: [178, 174, 167, 175, 175, 171, 177, 178, 177, 175, 175, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301564905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[2]R generation[0..12]: [99, 96, 92, 96, 96, 94, 98, 99, 98, 96, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301578145Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301591755Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301604425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301617485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301630015Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:45:24.301642625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"":"downstairs","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301664235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301676004Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"0","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301689004Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"1","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301701804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"client":"2","":"downstairs","session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301714794Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301726904Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"a4b30d20-fa7f-41af-a674-cc0ae0acb02f is now active with session: 3ae0d177-e291-4978-aec7-493e503df98e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301739324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301750974Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592,"session_id":"3ae0d177-e291-4978-aec7-493e503df98e"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301763023Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592} {"msg":"Activated sub_volume a4b30d20-fa7f-41af-a674-cc0ae0acb02f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:24.301773874Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 540 len 28672 data: 9 10 9 10 8 10 9 02/30 Read block 105 len 24576 03/30 Write block 288 len 24576 data: 9 10 11 11 10 9 04/30 Write block 441 len 12288 data: 5 7 7 05/30 Read block 75 len 16384 06/30 Read block 305 len 24576 07/30 Write block 396 len 40960 data: 15 18 14 14 14 13 13 13 10 7 08/30 Read block 100 len 32768 09/30 Write block 89 len 24576 data: 12 11 13 12 10 8 10/30 Read block 401 len 24576 11/30 Read block 140 len 4096 12/30 Write block 66 len 16384 data: 14 11 12 11 13/30 Flush 14/30 Write block 316 len 40960 data: 9 8 10 9 9 10 8 9 9 10 15/30 Write block 515 len 32768 data: 10 12 14 10 11 10 12 9 16/30 Read block 79 len 20480 17/30 Read block 540 len 28672 18/30 Read block 264 len 8192 19/30 Write block 402 len 20480 data: 14 14 11 8 7 20/30 Write block 358 len 12288 data: 9 9 9 21/30 Write block 141 len 36864 data: 6 6 5 5 6 6 6 6 6 22/30 Write block 54 len 4096 data: 10 23/30 Read block 339 len 16384 24/30 Write block 423 len 20480 data: 6 7 7 9 7 25/30 Read block 438 len 32768 26/30 Write block 194 len 28672 data: 6 4 6 7 9 9 8 27/30 Write block 332 len 4096 data: 10 28/30 Read block 361 len 4096 29/30 Read block 37 len 24576 30/30 Read block 141 len 12288 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:24.322782653Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 178 178 178 F F F 1 020-039 96 96 96 174 174 174 F F F 2 040-059 92 92 92 167 167 167 T T T 3 060-079 100 100 100 179 179 179 F F F 4 080-099 100 100 100 179 179 179 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 98 98 98 177 177 177 F F F 7 140-159 99 99 99 178 178 178 T T T 8 160-179 98 98 98 177 177 177 F F F 9 180-199 96 96 96 175 175 175 T T T 10 200-219 96 96 96 175 175 175 T T T 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 96 96 96 174 174 174 F F F 14 280-299 100 100 100 179 179 179 F F F 15 300-319 99 99 99 178 178 178 T T T 16 320-339 97 97 97 176 176 176 T T T 17 340-359 98 98 98 177 177 177 T T T 18 360-379 99 99 99 178 178 178 T T T 19 380-399 100 100 100 179 179 179 F F F 20 400-419 100 100 100 179 179 179 T T T 21 420-439 94 94 94 172 172 172 T T T 22 440-459 100 100 100 179 179 179 F F F 23 460-479 99 99 99 178 178 178 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 96 96 96 174 174 174 T T T 26 520-539 97 97 97 176 176 176 T T T 27 540-559 100 100 100 179 179 179 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 94 94 94 172 172 172 F F F Max gen: 100, Max flush: 179 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 178 178 178 F F F 1 020-039 96 96 96 174 174 174 F F F 2 040-059 101 101 101 180 180 180 F F F 3 060-079 100 100 100 179 179 179 F F F 4 080-099 100 100 100 179 179 179 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 98 98 98 177 177 177 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 98 98 98 177 177 177 F F F 9 180-199 101 101 101 180 180 180 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 96 96 96 174 174 174 F F F 14 280-299 100 100 100 179 179 179 F F F 15 300-319 101 101 101 180 180 180 F F F 16 320-339 101 101 101 180 180 180 F F F 17 340-359 101 101 101 180 180 180 F F F 18 360-379 101 101 101 180 180 180 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 101 101 101 180 180 180 F F F 21 420-439 101 101 101 180 180 180 F F F 22 440-459 100 100 100 179 179 179 F F F 23 460-479 99 99 99 178 178 178 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 101 101 101 180 180 180 F F F 26 520-539 101 101 101 180 180 180 F F F 27 540-559 100 100 100 179 179 179 F F F 28 560-579 88 88 88 158 158 158 F F F 29 580-599 94 94 94 172 172 172 F F F Max gen: 101, Max flush: 180 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.629919062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:28.676360842Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.676895517Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.709435653Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.710303445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.710347414Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.710364524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.710383444Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600} {"msg":"Crucible 52c70a48-653d-4e79-b33c-deb5f8927869 has session id: 7338fa88-0c2b-4375-bf22-c7aa80ac0c6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711056338Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Upstairs opts: Upstairs UUID: 52c70a48-653d-4e79-b33c-deb5f8927869, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711101678Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Crucible stats registered with UUID: 52c70a48-653d-4e79-b33c-deb5f8927869","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711125747Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:28.711146367Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711605283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711631983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600} {"msg":"52c70a48-653d-4e79-b33c-deb5f8927869 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711690062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711717582Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"0","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711750562Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"0","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711766172Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"1","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711780461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"1","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711794001Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"2","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711808431Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"2","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711822201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711845111Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711872401Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41350)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.71192551Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45649)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.711958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48835)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.712133308Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.712803702Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"1","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.712831852Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"2","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.712866192Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"0","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.713538846Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"1","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.713567805Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"2","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.713672484Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"0","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.71416303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714262189Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[0]R flush_numbers[0..12]: [178, 174, 180, 179, 179, 171, 177, 180, 177, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714331589Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[0]R generation[0..12]: [99, 96, 101, 100, 100, 94, 98, 101, 98, 101, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714351918Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714366908Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[1]R flush_numbers[0..12]: [178, 174, 180, 179, 179, 171, 177, 180, 177, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714390038Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[1]R generation[0..12]: [99, 96, 101, 100, 100, 94, 98, 101, 98, 101, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714415888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714443528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[2]R flush_numbers[0..12]: [178, 174, 180, 179, 179, 171, 177, 180, 177, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714469547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[2]R generation[0..12]: [99, 96, 101, 100, 100, 94, 98, 101, 98, 101, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714494867Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714515047Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:45:28.714528237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714546716Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714560056Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714572656Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"":"downstairs","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714585346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714596956Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"0","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714609816Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"1","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714622326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"client":"2","":"downstairs","session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714634886Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714650246Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"52c70a48-653d-4e79-b33c-deb5f8927869 is now active with session: 7338fa88-0c2b-4375-bf22-c7aa80ac0c6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714663146Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714674655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600,"session_id":"7338fa88-0c2b-4375-bf22-c7aa80ac0c6d"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714686335Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600} {"msg":"Activated sub_volume 52c70a48-653d-4e79-b33c-deb5f8927869","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:28.714697245Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 24576 data: 10 7 6 6 8 7 02/30 Write block 13 len 40960 data: 9 7 10 11 12 12 13 12 8 9 03/30 Read block 69 len 4096 04/30 Write block 575 len 24576 data: 7 6 10 11 14 12 05/30 Read block 523 len 24576 06/30 Write block 124 len 32768 data: 10 8 8 8 7 6 5 5 07/30 Write block 317 len 8192 data: 9 11 08/30 Write block 132 len 12288 data: 4 6 7 09/30 Read block 75 len 8192 10/30 Read block 296 len 24576 11/30 Write block 41 len 32768 data: 7 7 8 8 7 5 6 7 12/30 Read block 572 len 20480 13/30 Read block 463 len 28672 14/30 Read block 34 len 8192 15/30 Read block 433 len 36864 16/30 Read block 488 len 28672 17/30 Read block 268 len 28672 18/30 Flush 19/30 Write block 505 len 40960 data: 7 8 11 10 10 9 9 9 11 11 20/30 Flush 21/30 Write block 459 len 24576 data: 5 5 6 8 8 10 22/30 Write block 292 len 16384 data: 11 10 10 9 23/30 Read block 105 len 40960 24/30 Read block 129 len 12288 25/30 Flush 26/30 Write block 66 len 40960 data: 15 12 13 12 8 6 6 7 7 7 27/30 Write block 539 len 24576 data: 9 10 11 10 11 9 28/30 Read block 122 len 28672 29/30 Read block 427 len 8192 30/30 Write block 168 len 36864 data: 4 4 4 3 4 6 6 8 6 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:28.73623848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 181 181 181 F F F 1 020-039 102 102 102 181 181 181 F F F 2 040-059 102 102 102 181 181 181 F F F 3 060-079 100 100 100 179 179 179 T T T 4 080-099 100 100 100 179 179 179 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 102 102 102 181 181 181 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 98 98 98 177 177 177 T T T 9 180-199 101 101 101 180 180 180 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 96 96 96 174 174 174 F F F 14 280-299 102 102 102 183 183 183 F F F 15 300-319 102 102 102 181 181 181 F F F 16 320-339 101 101 101 180 180 180 F F F 17 340-359 102 102 102 181 181 181 F F F 18 360-379 101 101 101 180 180 180 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 101 101 101 180 180 180 F F F 21 420-439 101 101 101 180 180 180 F F F 22 440-459 102 102 102 183 183 183 F F F 23 460-479 102 102 102 183 183 183 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 102 102 102 182 182 182 F F F 26 520-539 101 101 101 180 180 180 T T T 27 540-559 100 100 100 179 179 179 T T T 28 560-579 102 102 102 181 181 181 F F F 29 580-599 102 102 102 181 181 181 F F F Max gen: 102, Max flush: 183 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 181 181 181 F F F 1 020-039 102 102 102 181 181 181 F F F 2 040-059 102 102 102 181 181 181 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 100 100 100 179 179 179 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 102 102 102 181 181 181 F F F 7 140-159 101 101 101 180 180 180 F F F 8 160-179 103 103 103 184 184 184 F F F 9 180-199 101 101 101 180 180 180 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 96 96 96 174 174 174 F F F 14 280-299 102 102 102 183 183 183 F F F 15 300-319 102 102 102 181 181 181 F F F 16 320-339 101 101 101 180 180 180 F F F 17 340-359 102 102 102 181 181 181 F F F 18 360-379 101 101 101 180 180 180 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 101 101 101 180 180 180 F F F 21 420-439 101 101 101 180 180 180 F F F 22 440-459 102 102 102 183 183 183 F F F 23 460-479 102 102 102 183 183 183 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 102 102 102 182 182 182 F F F 26 520-539 103 103 103 184 184 184 F F F 27 540-559 103 103 103 184 184 184 F F F 28 560-579 102 102 102 181 181 181 F F F 29 580-599 102 102 102 181 181 181 F F F Max gen: 103, Max flush: 184 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.004098944Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.047973598Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.048602472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.048653801Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.048671221Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.048688871Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608} {"msg":"Crucible 83dd2f66-72be-400a-beaa-f61fab0f93d2 has session id: f68f3689-0d4e-4807-b52d-5a3ff0366608","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.049300346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Upstairs opts: Upstairs UUID: 83dd2f66-72be-400a-beaa-f61fab0f93d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.049328275Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Crucible stats registered with UUID: 83dd2f66-72be-400a-beaa-f61fab0f93d2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.049342505Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:33.049354725Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.049832441Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.04985629Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608} {"msg":"83dd2f66-72be-400a-beaa-f61fab0f93d2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.04994088Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.049958009Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"0","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.049987439Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"0","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050001429Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"1","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050015689Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"1","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050032009Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"2","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050045979Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"2","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050121208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050153108Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050181167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"ds_connection connected from Ok(127.0.0.1:35442)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050206917Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"ds_connection connected from Ok(127.0.0.1:36701)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050230947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"ds_connection connected from Ok(127.0.0.1:55624)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.050398245Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.05097379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"2","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.05102879Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"0","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.051282828Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"1","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.051770513Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"2","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.051800633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"0","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.051816523Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"1","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052334188Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052443967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[0]R flush_numbers[0..12]: [181, 181, 181, 184, 179, 171, 181, 180, 184, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052506937Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[0]R generation[0..12]: [102, 102, 102, 103, 100, 94, 102, 101, 103, 101, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052535826Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052551286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[1]R flush_numbers[0..12]: [181, 181, 181, 184, 179, 171, 181, 180, 184, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052565826Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[1]R generation[0..12]: [102, 102, 102, 103, 100, 94, 102, 101, 103, 101, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052587566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052601876Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[2]R flush_numbers[0..12]: [181, 181, 181, 184, 179, 171, 181, 180, 184, 180, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052615465Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[2]R generation[0..12]: [102, 102, 102, 103, 100, 94, 102, 101, 103, 101, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052628845Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052642395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052655235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052677645Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052702565Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052732924Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"":"downstairs","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052768004Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052792214Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"0","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052818514Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"1","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052844803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"client":"2","":"downstairs","session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052872063Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052897073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"83dd2f66-72be-400a-beaa-f61fab0f93d2 is now active with session: f68f3689-0d4e-4807-b52d-5a3ff0366608","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052923243Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052945713Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608,"session_id":"f68f3689-0d4e-4807-b52d-5a3ff0366608"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:33.052969462Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608} {"msg":"Activated sub_volume 83dd2f66-72be-400a-beaa-f61fab0f93d2","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T01:45:33.052990642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 594 len 4096 data: 11 02/30 Write block 534 len 16384 data: 7 9 7 6 03/30 Read block 227 len 12288 04/30 Flush 05/30 Write block 313 len 4096 data: 7 06/30 Flush 07/30 Write block 419 len 4096 data: 8 08/30 Read block 375 len 24576 09/30 Read block 278 len 32768 10/30 Read block 533 len 8192 11/30 Read block 338 len 12288 12/30 Read block 405 len 24576 13/30 Read block 269 len 28672 14/30 Read block 69 len 36864 15/30 Write block 408 len 32768 data: 5 5 6 8 8 8 7 7 16/30 Read block 310 len 24576 17/30 Write block 458 len 16384 data: 8 6 6 7 18/30 Write block 122 len 4096 data: 12 19/30 Read block 163 len 12288 20/30 Write block 144 len 4096 data: 6 21/30 Write block 88 len 36864 data: 10 13 12 14 13 11 9 8 6 22/30 Write block 278 len 28672 data: 12 12 12 12 14 12 11 23/30 Flush 24/30 Write block 125 len 4096 data: 9 25/30 Write block 187 len 24576 data: 4 3 5 5 5 6 26/30 Read block 527 len 4096 27/30 Write block 302 len 40960 data: 9 9 8 6 6 7 8 11 9 8 28/30 Read block 516 len 40960 29/30 Read block 550 len 32768 30/30 Read block 566 len 8192 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:33.071861301Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 181 181 181 F F F 1 020-039 102 102 102 181 181 181 F F F 2 040-059 102 102 102 181 181 181 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 104 104 104 187 187 187 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 104 104 104 187 187 187 T T T 7 140-159 104 104 104 187 187 187 F F F 8 160-179 103 103 103 184 184 184 F F F 9 180-199 101 101 101 180 180 180 T T T 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 104 104 104 187 187 187 F F F 14 280-299 104 104 104 187 187 187 F F F 15 300-319 104 104 104 186 186 186 T T T 16 320-339 101 101 101 180 180 180 F F F 17 340-359 102 102 102 181 181 181 F F F 18 360-379 101 101 101 180 180 180 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 104 104 104 187 187 187 F F F 21 420-439 101 101 101 180 180 180 F F F 22 440-459 104 104 104 187 187 187 F F F 23 460-479 104 104 104 187 187 187 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 102 102 102 182 182 182 F F F 26 520-539 104 104 104 185 185 185 F F F 27 540-559 103 103 103 184 184 184 F F F 28 560-579 102 102 102 181 181 181 F F F 29 580-599 104 104 104 185 185 185 F F F Max gen: 104, Max flush: 187 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 181 181 181 F F F 1 020-039 102 102 102 181 181 181 F F F 2 040-059 102 102 102 181 181 181 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 104 104 104 187 187 187 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 105 105 105 188 188 188 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 103 103 103 184 184 184 F F F 9 180-199 105 105 105 188 188 188 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 104 104 104 187 187 187 F F F 14 280-299 104 104 104 187 187 187 F F F 15 300-319 105 105 105 188 188 188 F F F 16 320-339 101 101 101 180 180 180 F F F 17 340-359 102 102 102 181 181 181 F F F 18 360-379 101 101 101 180 180 180 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 104 104 104 187 187 187 F F F 21 420-439 101 101 101 180 180 180 F F F 22 440-459 104 104 104 187 187 187 F F F 23 460-479 104 104 104 187 187 187 F F F 24 480-499 96 96 96 174 174 174 F F F 25 500-519 102 102 102 182 182 182 F F F 26 520-539 104 104 104 185 185 185 F F F 27 540-559 103 103 103 184 184 184 F F F 28 560-579 102 102 102 181 181 181 F F F 29 580-599 104 104 104 185 185 185 F F F Max gen: 105, Max flush: 188 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.301192335Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.345652992Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.346253777Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.346303127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.346317716Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.346332086Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616} {"msg":"Crucible 6fb7f7be-ccca-4bfd-a87d-447c99c25b70 has session id: 59c7641d-c0ca-4e67-a33d-499a047a6825","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.346911211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Upstairs opts: Upstairs UUID: 6fb7f7be-ccca-4bfd-a87d-447c99c25b70, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.346948201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Crucible stats registered with UUID: 6fb7f7be-ccca-4bfd-a87d-447c99c25b70","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.346964361Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:37.34697746Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347530905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347566025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616} {"msg":"6fb7f7be-ccca-4bfd-a87d-447c99c25b70 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347619695Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347645794Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"0","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347684034Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"0","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347709804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"1","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347735734Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"1","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347761213Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"2","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347787613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"2","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347814033Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347840353Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347867732Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"ds_connection connected from Ok(127.0.0.1:54839)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347893752Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"ds_connection connected from Ok(127.0.0.1:53954)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347921442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"ds_connection connected from Ok(127.0.0.1:53430)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.347949932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.348618655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"0","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.348650845Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"2","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.34919777Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"1","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.349338789Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"2","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.349372659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"0","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.349731156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"1","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350024013Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350048593Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[0]R flush_numbers[0..12]: [181, 181, 181, 184, 187, 171, 188, 187, 184, 188, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350383999Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[0]R generation[0..12]: [102, 102, 102, 103, 104, 94, 105, 104, 103, 105, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350420559Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350447629Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[1]R flush_numbers[0..12]: [181, 181, 181, 184, 187, 171, 188, 187, 184, 188, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350473359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[1]R generation[0..12]: [102, 102, 102, 103, 104, 94, 105, 104, 103, 105, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350496838Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350521988Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[2]R flush_numbers[0..12]: [181, 181, 181, 184, 187, 171, 188, 187, 184, 188, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350554608Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[2]R generation[0..12]: [102, 102, 102, 103, 104, 94, 105, 104, 103, 105, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350583808Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350612757Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350646537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350672967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350698987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350724827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"":"downstairs","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350751176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350774826Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"0","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350804526Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"1","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350830946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"client":"2","":"downstairs","session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350846645Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350859205Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"6fb7f7be-ccca-4bfd-a87d-447c99c25b70 is now active with session: 59c7641d-c0ca-4e67-a33d-499a047a6825","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350871995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350883685Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616,"session_id":"59c7641d-c0ca-4e67-a33d-499a047a6825"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350899495Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616} {"msg":"Activated sub_volume 6fb7f7be-ccca-4bfd-a87d-447c99c25b70","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:37.350911825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 40960 data: 6 7 8 7 8 9 11 8 11 11 02/30 Read block 494 len 4096 03/30 Write block 275 len 4096 data: 9 04/30 Write block 81 len 4096 data: 13 05/30 Write block 483 len 28672 data: 7 7 7 8 6 6 10 06/30 Read block 30 len 28672 07/30 Write block 535 len 20480 data: 10 8 7 9 10 08/30 Read block 39 len 8192 09/30 Write block 4 len 36864 data: 5 7 9 12 12 10 10 11 10 10/30 Write block 264 len 28672 data: 9 9 7 9 9 8 9 11/30 Flush 12/30 Write block 587 len 24576 data: 9 8 6 7 8 10 13/30 Write block 265 len 4096 data: 10 14/30 Write block 439 len 12288 data: 7 7 6 15/30 Write block 369 len 28672 data: 5 5 8 10 10 12 11 16/30 Write block 433 len 4096 data: 6 17/30 Read block 389 len 20480 18/30 Flush 19/30 Flush 20/30 Write block 180 len 4096 data: 4 21/30 Write block 523 len 20480 data: 7 9 9 10 10 22/30 Write block 167 len 8192 data: 3 5 23/30 Write block 456 len 28672 data: 7 7 9 7 7 8 9 24/30 Read block 8 len 20480 25/30 Flush 26/30 Write block 276 len 36864 data: 8 10 13 13 13 13 15 13 12 27/30 Write block 372 len 24576 data: 11 11 13 12 10 11 28/30 Write block 43 len 8192 data: 9 9 29/30 Write block 120 len 40960 data: 11 12 13 12 11 10 9 9 8 7 30/30 Read block 253 len 36864 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:37.368198688Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 189 189 189 F F F 1 020-039 102 102 102 181 181 181 F F F 2 040-059 106 106 106 189 189 189 T T T 3 060-079 103 103 103 184 184 184 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 105 105 105 188 188 188 T T T 7 140-159 104 104 104 187 187 187 F F F 8 160-179 106 106 106 192 192 192 F F F 9 180-199 106 106 106 192 192 192 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 106 106 106 190 190 190 T T T 14 280-299 104 104 104 187 187 187 T T T 15 300-319 105 105 105 188 188 188 F F F 16 320-339 101 101 101 180 180 180 F F F 17 340-359 102 102 102 181 181 181 F F F 18 360-379 106 106 106 190 190 190 T T T 19 380-399 100 100 100 179 179 179 F F F 20 400-419 104 104 104 187 187 187 F F F 21 420-439 106 106 106 190 190 190 F F F 22 440-459 106 106 106 192 192 192 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 189 189 189 F F F 25 500-519 102 102 102 182 182 182 F F F 26 520-539 106 106 106 192 192 192 F F F 27 540-559 103 103 103 184 184 184 F F F 28 560-579 102 102 102 181 181 181 F F F 29 580-599 106 106 106 190 190 190 F F F Max gen: 106, Max flush: 192 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 189 189 189 F F F 1 020-039 102 102 102 181 181 181 F F F 2 040-059 107 107 107 193 193 193 F F F 3 060-079 103 103 103 184 184 184 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 94 94 94 171 171 171 F F F 6 120-139 107 107 107 193 193 193 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 106 106 106 192 192 192 F F F 9 180-199 106 106 106 192 192 192 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 107 107 107 193 193 193 F F F 14 280-299 107 107 107 193 193 193 F F F 15 300-319 105 105 105 188 188 188 F F F 16 320-339 101 101 101 180 180 180 F F F 17 340-359 102 102 102 181 181 181 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 104 104 104 187 187 187 F F F 21 420-439 106 106 106 190 190 190 F F F 22 440-459 106 106 106 192 192 192 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 189 189 189 F F F 25 500-519 102 102 102 182 182 182 F F F 26 520-539 106 106 106 192 192 192 F F F 27 540-559 103 103 103 184 184 184 F F F 28 560-579 102 102 102 181 181 181 F F F 29 580-599 106 106 106 190 190 190 F F F Max gen: 107, Max flush: 193 Begin loop 54 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.616205142Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.660570421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.661111186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.661156146Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.661171856Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.661190286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624} {"msg":"Crucible eddae176-9684-437d-94ee-c66d3faa83dc has session id: 977c2771-4c28-4f2f-93ba-cf8a95ff36be","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.6617603Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Upstairs opts: Upstairs UUID: eddae176-9684-437d-94ee-c66d3faa83dc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.66179865Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Crucible stats registered with UUID: eddae176-9684-437d-94ee-c66d3faa83dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.66182216Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:41.66184298Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662267506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662296016Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624} {"msg":"eddae176-9684-437d-94ee-c66d3faa83dc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662355675Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662381485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"0","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662412635Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"0","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662426724Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"1","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662440694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"1","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662453524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"2","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662467264Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"2","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662494854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662521563Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662546623Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"ds_connection connected from Ok(127.0.0.1:50201)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662572443Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"ds_connection connected from Ok(127.0.0.1:43064)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662597003Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"ds_connection connected from Ok(127.0.0.1:39433)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.662621533Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.663276067Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"1","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.663441665Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"0","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.663918611Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"1","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.664086829Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"0","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.664116549Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"2","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.664589485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.664661834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.664815983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"2","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[0]R flush_numbers[0..12]: [189, 181, 193, 184, 189, 171, 193, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665331858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[0]R generation[0..12]: [106, 102, 107, 103, 106, 94, 107, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665360278Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665376528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[1]R flush_numbers[0..12]: [189, 181, 193, 184, 189, 171, 193, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665390997Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[1]R generation[0..12]: [106, 102, 107, 103, 106, 94, 107, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665404887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665418787Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[2]R flush_numbers[0..12]: [189, 181, 193, 184, 189, 171, 193, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665432577Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[2]R generation[0..12]: [106, 102, 107, 103, 106, 94, 107, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665446097Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665459837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665472277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665485067Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665497607Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:45:41.665510056Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"":"downstairs","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665527766Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665539496Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"0","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665554266Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"1","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665567766Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"client":"2","":"downstairs","session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665580796Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665592726Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"eddae176-9684-437d-94ee-c66d3faa83dc is now active with session: 977c2771-4c28-4f2f-93ba-cf8a95ff36be","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665605186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665616886Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624,"session_id":"977c2771-4c28-4f2f-93ba-cf8a95ff36be"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665628665Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624} {"msg":"Activated sub_volume eddae176-9684-437d-94ee-c66d3faa83dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:41.665639405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 55 len 28672 data: 12 12 8 5 5 10 14 02/30 Write block 523 len 24576 data: 8 10 10 11 11 7 03/30 Write block 337 len 24576 data: 13 10 9 11 11 8 04/30 Write block 43 len 8192 data: 10 10 05/30 Write block 106 len 24576 data: 11 10 10 9 11 10 06/30 Read block 13 len 8192 07/30 Read block 234 len 20480 08/30 Flush 09/30 Write block 333 len 40960 data: 9 11 12 11 14 11 10 12 12 9 10/30 Read block 276 len 40960 11/30 Write block 301 len 8192 data: 7 10 12/30 Write block 129 len 4096 data: 8 13/30 Flush 14/30 Write block 264 len 16384 data: 10 11 8 10 15/30 Read block 163 len 12288 16/30 Write block 551 len 28672 data: 11 9 10 8 9 8 9 17/30 Read block 179 len 24576 18/30 Flush 19/30 Write block 319 len 16384 data: 10 10 11 9 20/30 Write block 276 len 40960 data: 9 11 14 14 14 14 16 14 13 7 21/30 Read block 560 len 16384 22/30 Read block 448 len 8192 23/30 Read block 468 len 24576 24/30 Read block 527 len 24576 25/30 Write block 13 len 36864 data: 10 8 11 12 13 13 14 13 9 26/30 Read block 238 len 24576 27/30 Write block 119 len 8192 data: 6 12 28/30 Read block 298 len 24576 29/30 Read block 553 len 36864 30/30 Write block 556 len 40960 data: 9 10 5 5 5 5 4 5 6 9 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:41.686263269Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 189 189 189 T T T 1 020-039 102 102 102 181 181 181 T T T 2 040-059 108 108 108 194 194 194 F F F 3 060-079 108 108 108 194 194 194 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 108 108 108 194 194 194 T T T 6 120-139 108 108 108 195 195 195 T T T 7 140-159 104 104 104 187 187 187 F F F 8 160-179 106 106 106 192 192 192 F F F 9 180-199 106 106 106 192 192 192 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 108 108 108 196 196 196 T T T 14 280-299 107 107 107 193 193 193 T T T 15 300-319 108 108 108 195 195 195 T T T 16 320-339 108 108 108 195 195 195 T T T 17 340-359 108 108 108 195 195 195 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 104 104 104 187 187 187 F F F 21 420-439 106 106 106 190 190 190 F F F 22 440-459 106 106 106 192 192 192 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 189 189 189 F F F 25 500-519 102 102 102 182 182 182 F F F 26 520-539 108 108 108 194 194 194 F F F 27 540-559 108 108 108 196 196 196 T T T 28 560-579 102 102 102 181 181 181 T T T 29 580-599 106 106 106 190 190 190 F F F Max gen: 108, Max flush: 196 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 197 197 197 F F F 1 020-039 109 109 109 197 197 197 F F F 2 040-059 108 108 108 194 194 194 F F F 3 060-079 108 108 108 194 194 194 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 109 109 109 197 197 197 F F F 6 120-139 109 109 109 197 197 197 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 106 106 106 192 192 192 F F F 9 180-199 106 106 106 192 192 192 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 94 94 94 171 171 171 F F F 13 260-279 109 109 109 197 197 197 F F F 14 280-299 109 109 109 197 197 197 F F F 15 300-319 109 109 109 197 197 197 F F F 16 320-339 109 109 109 197 197 197 F F F 17 340-359 108 108 108 195 195 195 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 104 104 104 187 187 187 F F F 21 420-439 106 106 106 190 190 190 F F F 22 440-459 106 106 106 192 192 192 F F F 23 460-479 106 106 106 192 192 192 F F F 24 480-499 106 106 106 189 189 189 F F F 25 500-519 102 102 102 182 182 182 F F F 26 520-539 108 108 108 194 194 194 F F F 27 540-559 109 109 109 197 197 197 F F F 28 560-579 109 109 109 197 197 197 F F F 29 580-599 106 106 106 190 190 190 F F F Max gen: 109, Max flush: 197 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.952138881Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.99749566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.998365532Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.998418952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.998438142Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.998456192Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632} {"msg":"Crucible 0ede5503-5c13-4c33-81d7-bd8276ef739b has session id: 0a51de3e-c7a1-4829-b57c-16804397af35","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999019946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Upstairs opts: Upstairs UUID: 0ede5503-5c13-4c33-81d7-bd8276ef739b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999057166Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Crucible stats registered with UUID: 0ede5503-5c13-4c33-81d7-bd8276ef739b","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999079456Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:45.999105896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999604531Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999631061Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632} {"msg":"0ede5503-5c13-4c33-81d7-bd8276ef739b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.9996987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.99971744Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"0","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.99974747Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"0","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.99976167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"1","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.99977809Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"1","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999814569Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"2","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999841659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"2","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999901169Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999938148Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:45.999966168Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"ds_connection connected from Ok(127.0.0.1:40722)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.000031537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"ds_connection connected from Ok(127.0.0.1:36520)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.000061807Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"ds_connection connected from Ok(127.0.0.1:63310)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.000088547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.00080585Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"0","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.00084637Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"2","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.00088175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"1","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.001381425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"0","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.001634943Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"2","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.001825681Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"1","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002028869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002149638Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[0]R flush_numbers[0..12]: [197, 197, 194, 194, 189, 197, 197, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002421416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[0]R generation[0..12]: [109, 109, 108, 108, 106, 109, 109, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002447155Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002463386Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[1]R flush_numbers[0..12]: [197, 197, 194, 194, 189, 197, 197, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002480825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[1]R generation[0..12]: [109, 109, 108, 108, 106, 109, 109, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002504935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002531545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[2]R flush_numbers[0..12]: [197, 197, 194, 194, 189, 197, 197, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002557385Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[2]R generation[0..12]: [109, 109, 108, 108, 106, 109, 109, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002583764Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002608444Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002627544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002657744Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002683473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002703683Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"":"downstairs","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002717293Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002729313Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"0","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002752333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"1","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002776932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"client":"2","":"downstairs","session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002802292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002816832Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"0ede5503-5c13-4c33-81d7-bd8276ef739b is now active with session: 0a51de3e-c7a1-4829-b57c-16804397af35","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002833272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002854822Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632,"session_id":"0a51de3e-c7a1-4829-b57c-16804397af35"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002876532Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632} {"msg":"Activated sub_volume 0ede5503-5c13-4c33-81d7-bd8276ef739b","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:46.002897582Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 404 len 12288 data: 12 9 8 02/30 Read block 184 len 16384 03/30 Read block 254 len 40960 04/30 Read block 81 len 36864 05/30 Write block 126 len 8192 data: 10 10 06/30 Read block 375 len 8192 07/30 Write block 241 len 40960 data: 10 9 8 7 7 6 5 6 6 6 08/30 Write block 297 len 24576 data: 6 6 4 7 8 11 09/30 Flush 10/30 Write block 478 len 4096 data: 8 11/30 Write block 112 len 40960 data: 8 9 7 4 4 6 5 7 13 13 12/30 Read block 4 len 20480 13/30 Write block 328 len 16384 data: 11 11 11 12 14/30 Read block 141 len 20480 15/30 Read block 268 len 12288 16/30 Flush 17/30 Write block 553 len 4096 data: 11 18/30 Write block 425 len 8192 data: 8 10 19/30 Read block 417 len 32768 20/30 Write block 281 len 4096 data: 15 21/30 Flush 22/30 Write block 500 len 20480 data: 7 8 7 8 8 23/30 Write block 105 len 32768 data: 9 12 11 11 10 12 11 9 24/30 Read block 373 len 12288 25/30 Read block 60 len 28672 26/30 Write block 531 len 8192 data: 6 7 27/30 Write block 103 len 28672 data: 10 6 10 13 12 12 11 28/30 Read block 141 len 40960 29/30 Read block 374 len 24576 30/30 Write block 511 len 16384 data: 10 10 12 12 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:46.022228927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 197 197 197 F F F 1 020-039 109 109 109 197 197 197 F F F 2 040-059 108 108 108 194 194 194 F F F 3 060-079 108 108 108 194 194 194 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 110 110 110 199 199 199 T T T 6 120-139 110 110 110 199 199 199 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 106 106 106 192 192 192 F F F 9 180-199 106 106 106 192 192 192 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 110 110 110 198 198 198 F F F 13 260-279 109 109 109 197 197 197 F F F 14 280-299 110 110 110 200 200 200 F F F 15 300-319 110 110 110 198 198 198 F F F 16 320-339 110 110 110 199 199 199 F F F 17 340-359 108 108 108 195 195 195 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 110 110 110 198 198 198 F F F 21 420-439 110 110 110 200 200 200 F F F 22 440-459 106 106 106 192 192 192 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 106 106 106 189 189 189 F F F 25 500-519 102 102 102 182 182 182 T T T 26 520-539 108 108 108 194 194 194 T T T 27 540-559 110 110 110 200 200 200 F F F 28 560-579 109 109 109 197 197 197 F F F 29 580-599 106 106 106 190 190 190 F F F Max gen: 110, Max flush: 200 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 109 109 109 197 197 197 F F F 1 020-039 109 109 109 197 197 197 F F F 2 040-059 108 108 108 194 194 194 F F F 3 060-079 108 108 108 194 194 194 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 111 111 111 201 201 201 F F F 6 120-139 110 110 110 199 199 199 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 106 106 106 192 192 192 F F F 9 180-199 106 106 106 192 192 192 F F F 10 200-219 101 101 101 180 180 180 F F F 11 220-239 96 96 96 175 175 175 F F F 12 240-259 110 110 110 198 198 198 F F F 13 260-279 109 109 109 197 197 197 F F F 14 280-299 110 110 110 200 200 200 F F F 15 300-319 110 110 110 198 198 198 F F F 16 320-339 110 110 110 199 199 199 F F F 17 340-359 108 108 108 195 195 195 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 100 100 100 179 179 179 F F F 20 400-419 110 110 110 198 198 198 F F F 21 420-439 110 110 110 200 200 200 F F F 22 440-459 106 106 106 192 192 192 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 106 106 106 189 189 189 F F F 25 500-519 111 111 111 201 201 201 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 110 110 110 200 200 200 F F F 28 560-579 109 109 109 197 197 197 F F F 29 580-599 106 106 106 190 190 190 F F F Max gen: 111, Max flush: 201 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.250732967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.296330515Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.297063978Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.297110758Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.297125977Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.297141577Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640} {"msg":"Crucible 42af10ab-853d-4112-bb90-7a3adfcdccd4 has session id: 0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.297736852Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Upstairs opts: Upstairs UUID: 42af10ab-853d-4112-bb90-7a3adfcdccd4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.297776602Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Crucible stats registered with UUID: 42af10ab-853d-4112-bb90-7a3adfcdccd4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.297803151Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:50.297824741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298248927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298273567Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640} {"msg":"42af10ab-853d-4112-bb90-7a3adfcdccd4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298333126Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298351486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"0","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298381136Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"0","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298395756Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"1","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298416386Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"1","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298439836Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"2","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298465265Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"2","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298490685Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298516545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298544465Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"ds_connection connected from Ok(127.0.0.1:45075)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298570954Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38951)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298596984Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"ds_connection connected from Ok(127.0.0.1:36657)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.298806892Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.299224438Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"0","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.299360197Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"2","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.299477186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"1","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.299972732Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"0","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300037101Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"2","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300056531Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"1","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300581796Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300605686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[0]R flush_numbers[0..12]: [197, 197, 194, 194, 189, 201, 199, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300665156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[0]R generation[0..12]: [109, 109, 108, 108, 106, 111, 110, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300685725Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300703575Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[1]R flush_numbers[0..12]: [197, 197, 194, 194, 189, 201, 199, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300717825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[1]R generation[0..12]: [109, 109, 108, 108, 106, 111, 110, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300731235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300744905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[2]R flush_numbers[0..12]: [197, 197, 194, 194, 189, 201, 199, 187, 192, 192, 180, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300758335Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[2]R generation[0..12]: [109, 109, 108, 108, 106, 111, 110, 104, 106, 106, 101, 96]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300771655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300785094Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300797664Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300810504Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300831034Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300844654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300864704Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300876614Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"0","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300889454Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"1","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300902313Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"client":"2","":"downstairs","session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300917613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300929833Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"42af10ab-853d-4112-bb90-7a3adfcdccd4 is now active with session: 0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300942273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300953903Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640,"session_id":"0dbeb3f8-2a7c-4c6b-9a9f-1aa8b7514fe4"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300965763Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640} {"msg":"Activated sub_volume 42af10ab-853d-4112-bb90-7a3adfcdccd4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:50.300976753Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 433 len 20480 data: 7 6 8 7 7 02/30 Read block 116 len 40960 03/30 Read block 279 len 28672 04/30 Write block 216 len 4096 data: 11 05/30 Write block 429 len 40960 data: 9 9 9 8 8 7 9 8 8 6 06/30 Write block 204 len 4096 data: 9 07/30 Write block 289 len 36864 data: 11 12 12 12 11 11 10 7 7 08/30 Write block 382 len 36864 data: 7 8 7 8 8 10 14 14 16 09/30 Read block 298 len 24576 10/30 Flush 11/30 Write block 103 len 40960 data: 11 7 11 14 13 13 12 13 12 10 12/30 Write block 3 len 8192 data: 5 6 13/30 Flush 14/30 Write block 262 len 16384 data: 6 6 11 12 15/30 Write block 28 len 12288 data: 13 10 9 16/30 Read block 501 len 32768 17/30 Write block 381 len 8192 data: 6 8 18/30 Write block 290 len 20480 data: 13 13 13 12 12 19/30 Read block 54 len 16384 20/30 Read block 223 len 12288 21/30 Write block 330 len 40960 data: 12 13 11 10 12 13 12 15 12 11 22/30 Write block 578 len 28672 data: 12 15 13 9 8 9 10 23/30 Read block 121 len 8192 24/30 Write block 3 len 20480 data: 6 7 8 10 13 25/30 Write block 236 len 20480 data: 9 8 10 9 13 26/30 Read block 253 len 28672 27/30 Read block 309 len 16384 28/30 Write block 75 len 4096 data: 8 29/30 Write block 169 len 28672 data: 5 5 4 5 7 7 9 30/30 Read block 75 len 24576 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 17 16 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:50.320969132Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 203 203 203 T T T 1 020-039 109 109 109 197 197 197 T T T 2 040-059 108 108 108 194 194 194 F F F 3 060-079 108 108 108 194 194 194 T T T 4 080-099 106 106 106 189 189 189 F F F 5 100-119 112 112 112 203 203 203 F F F 6 120-139 110 110 110 199 199 199 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 106 106 106 192 192 192 T T T 9 180-199 106 106 106 192 192 192 F F F 10 200-219 112 112 112 202 202 202 F F F 11 220-239 96 96 96 175 175 175 T T T 12 240-259 110 110 110 198 198 198 T T T 13 260-279 109 109 109 197 197 197 T T T 14 280-299 112 112 112 202 202 202 T T T 15 300-319 110 110 110 198 198 198 F F F 16 320-339 110 110 110 199 199 199 T T T 17 340-359 108 108 108 195 195 195 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 112 112 112 202 202 202 T T T 20 400-419 110 110 110 198 198 198 F F F 21 420-439 112 112 112 202 202 202 F F F 22 440-459 106 106 106 192 192 192 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 106 106 106 189 189 189 F F F 25 500-519 111 111 111 201 201 201 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 110 110 110 200 200 200 F F F 28 560-579 109 109 109 197 197 197 T T T 29 580-599 106 106 106 190 190 190 T T T Max gen: 112, Max flush: 203 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 204 204 204 F F F 1 020-039 113 113 113 204 204 204 F F F 2 040-059 108 108 108 194 194 194 F F F 3 060-079 113 113 113 204 204 204 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 112 112 112 203 203 203 F F F 6 120-139 110 110 110 199 199 199 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 113 113 113 204 204 204 F F F 9 180-199 106 106 106 192 192 192 F F F 10 200-219 112 112 112 202 202 202 F F F 11 220-239 113 113 113 204 204 204 F F F 12 240-259 113 113 113 204 204 204 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 113 113 113 204 204 204 F F F 15 300-319 110 110 110 198 198 198 F F F 16 320-339 113 113 113 204 204 204 F F F 17 340-359 108 108 108 195 195 195 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 113 113 113 204 204 204 F F F 20 400-419 110 110 110 198 198 198 F F F 21 420-439 112 112 112 202 202 202 F F F 22 440-459 106 106 106 192 192 192 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 106 106 106 189 189 189 F F F 25 500-519 111 111 111 201 201 201 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 110 110 110 200 200 200 F F F 28 560-579 113 113 113 204 204 204 F F F 29 580-599 113 113 113 204 204 204 F F F Max gen: 113, Max flush: 204 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.597277229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.641936556Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.64253744Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.64258172Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.642596239Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.642611229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648} {"msg":"Crucible a0ae4b6a-206b-40f2-a3f1-6f7956ebe840 has session id: eff67fe1-23b9-452b-9102-5ebad42daa90","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643112285Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Upstairs opts: Upstairs UUID: a0ae4b6a-206b-40f2-a3f1-6f7956ebe840, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643142525Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Crucible stats registered with UUID: a0ae4b6a-206b-40f2-a3f1-6f7956ebe840","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643159785Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:54.643181964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.64364557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.64367946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648} {"msg":"a0ae4b6a-206b-40f2-a3f1-6f7956ebe840 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643734229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643768659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"0","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643817888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"0","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643848648Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643876268Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643898858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"2","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643923718Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"2","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643947287Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643973897Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.643998797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"ds_connection connected from Ok(127.0.0.1:35426)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.644025947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"ds_connection connected from Ok(127.0.0.1:52773)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.644185875Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:54.644285934Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:54.644318584Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:45:54.644343274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.644426683Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.644626691Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"2","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.644650771Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"0","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.645219116Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"0","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.645328505Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"2","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.645724661Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:45:54.64586996Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.645424134Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"ds_connection connected from Ok(127.0.0.1:56802)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.645512003Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.646647473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647183508Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[0]R flush_numbers[0..12]: [204, 204, 194, 204, 189, 203, 199, 187, 204, 192, 202, 204]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647773413Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[0]R generation[0..12]: [113, 113, 108, 113, 106, 112, 110, 104, 113, 106, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647799412Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647820132Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[1]R flush_numbers[0..12]: [204, 204, 194, 204, 189, 203, 199, 187, 204, 192, 202, 204]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647843292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[1]R generation[0..12]: [113, 113, 108, 113, 106, 112, 110, 104, 113, 106, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647862282Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647879792Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[2]R flush_numbers[0..12]: [204, 204, 194, 204, 189, 203, 199, 187, 204, 192, 202, 204]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647896282Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[2]R generation[0..12]: [113, 113, 108, 113, 106, 112, 110, 104, 113, 106, 112, 113]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647913031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647930201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647945921Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647961941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647983531Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.647999701Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"":"downstairs","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.648016141Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.64803082Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"0","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.64804735Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"1","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.64806329Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"client":"2","":"downstairs","session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.64808167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.64809655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"a0ae4b6a-206b-40f2-a3f1-6f7956ebe840 is now active with session: eff67fe1-23b9-452b-9102-5ebad42daa90","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.64811211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.64812666Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648,"session_id":"eff67fe1-23b9-452b-9102-5ebad42daa90"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T01:46:04.648142029Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648} {"msg":"Activated sub_volume a0ae4b6a-206b-40f2-a3f1-6f7956ebe840","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:04.648156189Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 41 len 12288 data: 8 8 11 02/30 Write block 52 len 12288 data: 12 9 12 03/30 Read block 440 len 8192 04/30 Read block 121 len 4096 05/30 Read block 388 len 36864 06/30 Write block 252 len 32768 data: 9 10 10 9 9 10 9 6 07/30 Read block 548 len 4096 08/30 Read block 122 len 16384 09/30 Write block 309 len 8192 data: 12 10 10/30 Write block 563 len 20480 data: 6 7 10 8 9 11/30 Read block 304 len 20480 12/30 Flush 13/30 Write block 329 len 36864 data: 12 13 14 12 11 13 14 13 16 14/30 Write block 348 len 24576 data: 7 7 10 10 9 9 15/30 Write block 499 len 20480 data: 6 8 9 8 9 16/30 Write block 395 len 12288 data: 11 16 19 17/30 Read block 63 len 40960 18/30 Read block 141 len 4096 19/30 Write block 426 len 20480 data: 11 8 9 10 10 20/30 Read block 442 len 32768 21/30 Write block 179 len 20480 data: 7 5 6 7 5 22/30 Write block 573 len 28672 data: 5 7 8 7 11 13 16 23/30 Write block 444 len 24576 data: 11 10 10 11 11 10 24/30 Write block 452 len 20480 data: 10 8 7 8 8 25/30 Write block 220 len 16384 data: 13 14 13 12 26/30 Write block 28 len 36864 data: 14 11 10 8 9 8 7 7 8 27/30 Write block 497 len 8192 data: 7 7 28/30 Read block 495 len 32768 29/30 Write block 325 len 20480 data: 11 8 10 12 13 30/30 Read block 478 len 36864 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 9 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 18 16 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:04.666760711Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 204 204 204 F F F 1 020-039 113 113 113 204 204 204 T T T 2 040-059 114 114 114 205 205 205 F F F 3 060-079 113 113 113 204 204 204 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 112 112 112 203 203 203 F F F 6 120-139 110 110 110 199 199 199 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 113 113 113 204 204 204 T T T 9 180-199 106 106 106 192 192 192 T T T 10 200-219 112 112 112 202 202 202 F F F 11 220-239 113 113 113 204 204 204 T T T 12 240-259 114 114 114 205 205 205 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 113 113 113 204 204 204 F F F 15 300-319 114 114 114 205 205 205 F F F 16 320-339 113 113 113 204 204 204 T T T 17 340-359 108 108 108 195 195 195 T T T 18 360-379 107 107 107 193 193 193 F F F 19 380-399 113 113 113 204 204 204 T T T 20 400-419 110 110 110 198 198 198 F F F 21 420-439 112 112 112 202 202 202 T T T 22 440-459 106 106 106 192 192 192 T T T 23 460-479 110 110 110 199 199 199 F F F 24 480-499 106 106 106 189 189 189 T T T 25 500-519 111 111 111 201 201 201 T T T 26 520-539 111 111 111 201 201 201 F F F 27 540-559 110 110 110 200 200 200 F F F 28 560-579 114 114 114 205 205 205 T T T 29 580-599 113 113 113 204 204 204 F F F Max gen: 114, Max flush: 205 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 204 204 204 F F F 1 020-039 115 115 115 206 206 206 F F F 2 040-059 114 114 114 205 205 205 F F F 3 060-079 113 113 113 204 204 204 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 112 112 112 203 203 203 F F F 6 120-139 110 110 110 199 199 199 F F F 7 140-159 104 104 104 187 187 187 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 115 115 115 206 206 206 F F F 10 200-219 112 112 112 202 202 202 F F F 11 220-239 115 115 115 206 206 206 F F F 12 240-259 114 114 114 205 205 205 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 113 113 113 204 204 204 F F F 15 300-319 114 114 114 205 205 205 F F F 16 320-339 115 115 115 206 206 206 F F F 17 340-359 115 115 115 206 206 206 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 115 115 115 206 206 206 F F F 20 400-419 110 110 110 198 198 198 F F F 21 420-439 115 115 115 206 206 206 F F F 22 440-459 115 115 115 206 206 206 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 115 115 115 206 206 206 F F F 25 500-519 115 115 115 206 206 206 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 110 110 110 200 200 200 F F F 28 560-579 115 115 115 206 206 206 F F F 29 580-599 113 113 113 204 204 204 F F F Max gen: 115, Max flush: 206 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:18.946456818Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:18.990288261Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:18.990728277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.022966916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.023552271Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.02358781Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.0236029Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.02361758Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656} {"msg":"Crucible b0516bda-6adb-49f2-b6fd-cc3c9f9f84f7 has session id: 237b65bd-a7cf-4f01-80e9-97a514ef45f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024274044Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Upstairs opts: Upstairs UUID: b0516bda-6adb-49f2-b6fd-cc3c9f9f84f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024315133Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Crucible stats registered with UUID: b0516bda-6adb-49f2-b6fd-cc3c9f9f84f7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024329953Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:19.024342413Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.02475996Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024790449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656} {"msg":"b0516bda-6adb-49f2-b6fd-cc3c9f9f84f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024820069Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024833199Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"0","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024862388Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"0","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024876268Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"1","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024889988Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"1","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024903068Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"2","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024916818Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"2","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024929478Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024943658Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.024967208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63178)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.025033357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:48842)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.025066307Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50819)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.025092577Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.025862779Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"1","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.025896769Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"2","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.026050048Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"0","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.026610983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"2","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.026640372Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"1","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.026667332Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"0","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027244457Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027353516Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[0]R flush_numbers[0..12]: [204, 206, 205, 204, 189, 203, 199, 187, 206, 206, 202, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027426875Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[0]R generation[0..12]: [113, 115, 114, 113, 106, 112, 110, 104, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027447075Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027462005Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[1]R flush_numbers[0..12]: [204, 206, 205, 204, 189, 203, 199, 187, 206, 206, 202, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027475545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[1]R generation[0..12]: [113, 115, 114, 113, 106, 112, 110, 104, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027489095Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027502825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[2]R flush_numbers[0..12]: [204, 206, 205, 204, 189, 203, 199, 187, 206, 206, 202, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027516355Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[2]R generation[0..12]: [113, 115, 114, 113, 106, 112, 110, 104, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027533224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027547064Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027559704Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027572484Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027585054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027597804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"":"downstairs","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:46:19.027610534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027628843Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"0","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027641714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"1","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027654353Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"client":"2","":"downstairs","session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027668003Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027680193Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"b0516bda-6adb-49f2-b6fd-cc3c9f9f84f7 is now active with session: 237b65bd-a7cf-4f01-80e9-97a514ef45f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027692633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027704323Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656,"session_id":"237b65bd-a7cf-4f01-80e9-97a514ef45f0"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027716503Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656} {"msg":"Activated sub_volume b0516bda-6adb-49f2-b6fd-cc3c9f9f84f7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:19.027727383Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 393 len 4096 data: 14 02/30 Write block 510 len 40960 data: 10 11 11 13 13 11 13 15 11 12 03/30 Write block 330 len 32768 data: 14 15 13 12 14 15 14 17 04/30 Read block 150 len 12288 05/30 Flush 06/30 Write block 148 len 16384 data: 7 7 6 7 07/30 Read block 270 len 32768 08/30 Write block 253 len 16384 data: 11 11 10 10 09/30 Read block 174 len 40960 10/30 Write block 100 len 8192 data: 9 10 11/30 Read block 312 len 20480 12/30 Write block 117 len 12288 data: 7 6 8 13/30 Read block 424 len 4096 14/30 Write block 247 len 24576 data: 6 7 7 7 8 10 15/30 Read block 192 len 16384 16/30 Read block 266 len 24576 17/30 Write block 488 len 16384 data: 7 11 7 7 18/30 Write block 133 len 28672 data: 7 8 7 6 9 7 6 19/30 Write block 551 len 16384 data: 12 10 12 9 20/30 Write block 411 len 40960 data: 9 9 9 8 8 9 8 9 9 9 21/30 Write block 589 len 24576 data: 7 8 9 11 13 12 22/30 Read block 490 len 32768 23/30 Read block 42 len 8192 24/30 Read block 77 len 8192 25/30 Flush 26/30 Write block 144 len 28672 data: 7 7 7 7 8 8 7 27/30 Write block 115 len 40960 data: 5 5 8 7 9 14 14 14 13 12 28/30 Write block 500 len 24576 data: 9 10 9 10 9 8 29/30 Write block 416 len 24576 data: 10 9 10 10 10 7 30/30 Write block 62 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:19.04675315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 204 204 204 F F F 1 020-039 115 115 115 206 206 206 F F F 2 040-059 114 114 114 205 205 205 F F F 3 060-079 113 113 113 204 204 204 T T T 4 080-099 106 106 106 189 189 189 F F F 5 100-119 116 116 116 208 208 208 T T T 6 120-139 116 116 116 208 208 208 T T T 7 140-159 116 116 116 208 208 208 T T T 8 160-179 115 115 115 206 206 206 F F F 9 180-199 115 115 115 206 206 206 F F F 10 200-219 112 112 112 202 202 202 F F F 11 220-239 115 115 115 206 206 206 F F F 12 240-259 116 116 116 208 208 208 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 113 113 113 204 204 204 F F F 15 300-319 114 114 114 205 205 205 F F F 16 320-339 116 116 116 207 207 207 F F F 17 340-359 115 115 115 206 206 206 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 116 116 116 207 207 207 F F F 20 400-419 116 116 116 208 208 208 T T T 21 420-439 116 116 116 208 208 208 T T T 22 440-459 115 115 115 206 206 206 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 116 116 116 208 208 208 F F F 25 500-519 116 116 116 207 207 207 T T T 26 520-539 111 111 111 201 201 201 F F F 27 540-559 116 116 116 208 208 208 F F F 28 560-579 115 115 115 206 206 206 F F F 29 580-599 116 116 116 208 208 208 F F F Max gen: 116, Max flush: 208 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 204 204 204 F F F 1 020-039 115 115 115 206 206 206 F F F 2 040-059 114 114 114 205 205 205 F F F 3 060-079 117 117 117 209 209 209 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 117 117 117 209 209 209 F F F 6 120-139 117 117 117 209 209 209 F F F 7 140-159 117 117 117 209 209 209 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 115 115 115 206 206 206 F F F 10 200-219 112 112 112 202 202 202 F F F 11 220-239 115 115 115 206 206 206 F F F 12 240-259 116 116 116 208 208 208 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 113 113 113 204 204 204 F F F 15 300-319 114 114 114 205 205 205 F F F 16 320-339 116 116 116 207 207 207 F F F 17 340-359 115 115 115 206 206 206 F F F 18 360-379 107 107 107 193 193 193 F F F 19 380-399 116 116 116 207 207 207 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 117 117 117 209 209 209 F F F 22 440-459 115 115 115 206 206 206 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 116 116 116 208 208 208 F F F 25 500-519 117 117 117 209 209 209 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 116 116 116 208 208 208 F F F 28 560-579 115 115 115 206 206 206 F F F 29 580-599 116 116 116 208 208 208 F F F Max gen: 117, Max flush: 209 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.328050194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:23.370209242Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.370588389Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.402152093Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.402840497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.402876407Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.402895256Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.402909816Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664} {"msg":"Crucible c87230d9-a11f-44ad-a72d-0d37f4a1c083 has session id: 0f415139-6798-467a-a9f7-b627ee11fa63","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.403519021Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Upstairs opts: Upstairs UUID: c87230d9-a11f-44ad-a72d-0d37f4a1c083, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.40355721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Crucible stats registered with UUID: c87230d9-a11f-44ad-a72d-0d37f4a1c083","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.40358219Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:23.40360383Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.403988737Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404021506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664} {"msg":"c87230d9-a11f-44ad-a72d-0d37f4a1c083 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404075056Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404102556Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"0","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404134285Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"0","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404149125Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"1","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404173235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"1","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404195725Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"2","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404225934Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"2","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404249604Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404273394Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404296234Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"ds_connection connected from Ok(127.0.0.1:48521)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404319743Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"ds_connection connected from Ok(127.0.0.1:45026)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404342623Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"ds_connection connected from Ok(127.0.0.1:55094)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.404368293Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.405127566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"2","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.405162406Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"1","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.405180576Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"0","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.40584001Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"1","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.405881229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"2","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.405953509Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"0","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406498424Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406525514Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[0]R flush_numbers[0..12]: [204, 206, 205, 209, 189, 209, 209, 209, 206, 206, 202, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406590033Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[0]R generation[0..12]: [113, 115, 114, 117, 106, 117, 117, 117, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406610613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406625943Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[1]R flush_numbers[0..12]: [204, 206, 205, 209, 189, 209, 209, 209, 206, 206, 202, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406639913Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[1]R generation[0..12]: [113, 115, 114, 117, 106, 117, 117, 117, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406654873Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406668952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[2]R flush_numbers[0..12]: [204, 206, 205, 209, 189, 209, 209, 209, 206, 206, 202, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406683092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[2]R generation[0..12]: [113, 115, 114, 117, 106, 117, 117, 117, 115, 115, 112, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406696612Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406710982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406723642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406737252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406750532Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406763072Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"":"downstairs","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406776331Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406791511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"0","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406805461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"1","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406818871Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"client":"2","":"downstairs","session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:46:23.406832521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406851471Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"c87230d9-a11f-44ad-a72d-0d37f4a1c083 is now active with session: 0f415139-6798-467a-a9f7-b627ee11fa63","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.406864551Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.40687626Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664,"session_id":"0f415139-6798-467a-a9f7-b627ee11fa63"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.40688897Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664} {"msg":"Activated sub_volume c87230d9-a11f-44ad-a72d-0d37f4a1c083","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:23.40689997Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 24576 data: 9 13 11 9 8 8 02/30 Read block 231 len 8192 03/30 Read block 102 len 20480 04/30 Read block 402 len 8192 05/30 Read block 22 len 16384 06/30 Read block 461 len 36864 07/30 Read block 23 len 16384 08/30 Read block 137 len 16384 09/30 Flush 10/30 Write block 208 len 16384 data: 12 14 11 9 11/30 Read block 546 len 28672 12/30 Read block 262 len 20480 13/30 Read block 126 len 24576 14/30 Write block 383 len 20480 data: 9 8 9 9 11 15/30 Read block 385 len 8192 16/30 Read block 309 len 8192 17/30 Read block 143 len 32768 18/30 Read block 544 len 12288 19/30 Write block 375 len 16384 data: 13 11 12 12 20/30 Read block 47 len 16384 21/30 Flush 22/30 Write block 13 len 4096 data: 11 23/30 Flush 24/30 Write block 572 len 40960 data: 8 6 8 9 8 12 14 17 14 10 25/30 Read block 123 len 24576 26/30 Read block 469 len 24576 27/30 Read block 363 len 28672 28/30 Read block 423 len 20480 29/30 Write block 150 len 16384 data: 8 8 8 8 30/30 Write block 208 len 32768 data: 13 15 12 10 9 9 10 8 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:23.427192007Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 212 212 212 F F F 1 020-039 115 115 115 206 206 206 F F F 2 040-059 114 114 114 205 205 205 F F F 3 060-079 117 117 117 209 209 209 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 117 117 117 209 209 209 F F F 6 120-139 117 117 117 209 209 209 F F F 7 140-159 117 117 117 209 209 209 T T T 8 160-179 115 115 115 206 206 206 F F F 9 180-199 115 115 115 206 206 206 F F F 10 200-219 118 118 118 211 211 211 T T T 11 220-239 115 115 115 206 206 206 F F F 12 240-259 116 116 116 208 208 208 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 113 113 113 204 204 204 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 116 116 116 207 207 207 F F F 17 340-359 115 115 115 206 206 206 F F F 18 360-379 118 118 118 211 211 211 F F F 19 380-399 118 118 118 211 211 211 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 117 117 117 209 209 209 F F F 22 440-459 115 115 115 206 206 206 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 116 116 116 208 208 208 F F F 25 500-519 117 117 117 209 209 209 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 116 116 116 208 208 208 F F F 28 560-579 115 115 115 206 206 206 T T T 29 580-599 116 116 116 208 208 208 T T T Max gen: 118, Max flush: 212 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 212 212 212 F F F 1 020-039 115 115 115 206 206 206 F F F 2 040-059 114 114 114 205 205 205 F F F 3 060-079 117 117 117 209 209 209 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 117 117 117 209 209 209 F F F 6 120-139 117 117 117 209 209 209 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 115 115 115 206 206 206 F F F 10 200-219 119 119 119 213 213 213 F F F 11 220-239 115 115 115 206 206 206 F F F 12 240-259 116 116 116 208 208 208 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 113 113 113 204 204 204 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 116 116 116 207 207 207 F F F 17 340-359 115 115 115 206 206 206 F F F 18 360-379 118 118 118 211 211 211 F F F 19 380-399 118 118 118 211 211 211 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 117 117 117 209 209 209 F F F 22 440-459 115 115 115 206 206 206 F F F 23 460-479 110 110 110 199 199 199 F F F 24 480-499 116 116 116 208 208 208 F F F 25 500-519 117 117 117 209 209 209 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 116 116 116 208 208 208 F F F 28 560-579 119 119 119 213 213 213 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 119, Max flush: 213 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.69190024Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:27.735862092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.736267108Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.767600744Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.768198609Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.768234639Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.768248359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.768262858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672} {"msg":"Crucible f4859429-5156-4228-a96c-2d604399f232 has session id: 4a98075c-0586-43a2-8451-35e2dc81aae2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.768765584Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Upstairs opts: Upstairs UUID: f4859429-5156-4228-a96c-2d604399f232, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.768801914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Crucible stats registered with UUID: f4859429-5156-4228-a96c-2d604399f232","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.768818364Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:27.768831883Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769280799Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769310979Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672} {"msg":"f4859429-5156-4228-a96c-2d604399f232 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769364348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769388778Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"0","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769418948Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"0","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769432798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"1","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769452428Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"1","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769475098Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"2","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769531727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"2","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769567187Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769595106Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769622676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:38277)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769654576Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:52744)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769687056Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"ds_connection connected from Ok(127.0.0.1:56964)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.769844794Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.770516908Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"2","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.770567508Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"1","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.770602417Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"0","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.771345711Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"1","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.7713773Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"0","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.77140381Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"2","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.771954145Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772080964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[0]R flush_numbers[0..12]: [212, 206, 205, 209, 189, 209, 209, 213, 206, 206, 213, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772127783Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[0]R generation[0..12]: [118, 115, 114, 117, 106, 117, 117, 119, 115, 115, 119, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772146373Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772160773Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[1]R flush_numbers[0..12]: [212, 206, 205, 209, 189, 209, 209, 213, 206, 206, 213, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772174533Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[1]R generation[0..12]: [118, 115, 114, 117, 106, 117, 117, 119, 115, 115, 119, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772188713Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772202643Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[2]R flush_numbers[0..12]: [212, 206, 205, 209, 189, 209, 209, 213, 206, 206, 213, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772216623Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[2]R generation[0..12]: [118, 115, 114, 117, 106, 117, 117, 119, 115, 115, 119, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772234293Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772248442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772261442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772274172Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772286782Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772307252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"":"downstairs","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772319982Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772331632Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"0","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772344492Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"1","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772357011Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"client":"2","":"downstairs","session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772369771Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772381601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"f4859429-5156-4228-a96c-2d604399f232 is now active with session: 4a98075c-0586-43a2-8451-35e2dc81aae2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772393811Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772405431Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672,"session_id":"4a98075c-0586-43a2-8451-35e2dc81aae2"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772417421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672} {"msg":"Activated sub_volume f4859429-5156-4228-a96c-2d604399f232","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:27.772430551Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 372 len 8192 data: 12 12 02/30 Read block 197 len 36864 03/30 Write block 552 len 36864 data: 11 13 10 10 10 11 6 6 6 04/30 Read block 533 len 12288 05/30 Read block 531 len 20480 06/30 Write block 252 len 20480 data: 11 12 12 11 11 07/30 Flush 08/30 Write block 497 len 16384 data: 8 8 7 10 09/30 Flush 10/30 Write block 465 len 16384 data: 11 11 11 12 11/30 Read block 184 len 28672 12/30 Read block 576 len 40960 13/30 Write block 294 len 24576 data: 13 11 8 8 7 5 14/30 Read block 440 len 24576 15/30 Write block 226 len 12288 data: 6 7 8 16/30 Read block 364 len 20480 17/30 Read block 352 len 4096 18/30 Read block 494 len 40960 19/30 Write block 280 len 36864 data: 15 16 17 15 14 8 9 9 10 20/30 Read block 529 len 4096 21/30 Read block 245 len 12288 22/30 Read block 202 len 4096 23/30 Write block 118 len 4096 data: 8 24/30 Write block 324 len 16384 data: 10 12 9 11 25/30 Write block 451 len 12288 data: 8 11 9 26/30 Read block 141 len 28672 27/30 Write block 38 len 40960 data: 7 7 8 9 9 12 11 8 7 8 28/30 Read block 104 len 4096 29/30 Read block 184 len 24576 30/30 Read block 14 len 32768 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 4 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:27.793981346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1672} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 212 212 212 F F F 1 020-039 115 115 115 206 206 206 T T T 2 040-059 114 114 114 205 205 205 T T T 3 060-079 117 117 117 209 209 209 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 117 117 117 209 209 209 T T T 6 120-139 117 117 117 209 209 209 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 115 115 115 206 206 206 F F F 10 200-219 119 119 119 213 213 213 F F F 11 220-239 115 115 115 206 206 206 T T T 12 240-259 120 120 120 214 214 214 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 113 113 113 204 204 204 T T T 15 300-319 118 118 118 210 210 210 F F F 16 320-339 116 116 116 207 207 207 T T T 17 340-359 115 115 115 206 206 206 F F F 18 360-379 120 120 120 214 214 214 F F F 19 380-399 118 118 118 211 211 211 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 117 117 117 209 209 209 F F F 22 440-459 115 115 115 206 206 206 T T T 23 460-479 110 110 110 199 199 199 T T T 24 480-499 120 120 120 215 215 215 F F F 25 500-519 120 120 120 215 215 215 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 120 120 120 214 214 214 F F F 28 560-579 120 120 120 214 214 214 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 120, Max flush: 215 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 212 212 212 F F F 1 020-039 121 121 121 216 216 216 F F F 2 040-059 121 121 121 216 216 216 F F F 3 060-079 117 117 117 209 209 209 F F F 4 080-099 106 106 106 189 189 189 F F F 5 100-119 121 121 121 216 216 216 F F F 6 120-139 117 117 117 209 209 209 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 115 115 115 206 206 206 F F F 10 200-219 119 119 119 213 213 213 F F F 11 220-239 121 121 121 216 216 216 F F F 12 240-259 120 120 120 214 214 214 F F F 13 260-279 113 113 113 204 204 204 F F F 14 280-299 121 121 121 216 216 216 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 115 115 115 206 206 206 F F F 18 360-379 120 120 120 214 214 214 F F F 19 380-399 118 118 118 211 211 211 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 117 117 117 209 209 209 F F F 22 440-459 121 121 121 216 216 216 F F F 23 460-479 121 121 121 216 216 216 F F F 24 480-499 120 120 120 215 215 215 F F F 25 500-519 120 120 120 215 215 215 F F F 26 520-539 111 111 111 201 201 201 F F F 27 540-559 120 120 120 214 214 214 F F F 28 560-579 120 120 120 214 214 214 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 121, Max flush: 216 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.080757359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.125231227Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.125839221Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.125892521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.1259165Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.125936Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680} {"msg":"Crucible 38301b99-015b-421e-84e5-dc4a887c4f17 has session id: e426a78d-a249-4f5a-9e39-af3b007b9764","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.126500445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Upstairs opts: Upstairs UUID: 38301b99-015b-421e-84e5-dc4a887c4f17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.126540485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Crucible stats registered with UUID: 38301b99-015b-421e-84e5-dc4a887c4f17","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.126562664Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:32.126581394Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.12705161Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.12708432Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680} {"msg":"38301b99-015b-421e-84e5-dc4a887c4f17 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127106479Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127126599Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"0","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127163549Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"0","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127185259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"1","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127207809Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"1","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127232778Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"2","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127255408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"2","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127277078Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127299788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127321897Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"ds_connection connected from Ok(127.0.0.1:65338)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127344547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"ds_connection connected from Ok(127.0.0.1:61337)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127368307Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"ds_connection connected from Ok(127.0.0.1:51884)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127392057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.127979642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"1","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.12813074Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"0","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.128474167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"1","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.128578386Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"0","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.129037682Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.129152201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.129751286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"2","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130270861Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"2","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[0]R flush_numbers[0..12]: [212, 216, 216, 209, 189, 216, 209, 213, 206, 206, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130791486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[0]R generation[0..12]: [118, 121, 121, 117, 106, 121, 117, 119, 115, 115, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130811676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130826106Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[1]R flush_numbers[0..12]: [212, 216, 216, 209, 189, 216, 209, 213, 206, 206, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130839676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[1]R generation[0..12]: [118, 121, 121, 117, 106, 121, 117, 119, 115, 115, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130853236Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130866956Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[2]R flush_numbers[0..12]: [212, 216, 216, 209, 189, 216, 209, 213, 206, 206, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130880385Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[2]R generation[0..12]: [118, 121, 121, 117, 106, 121, 117, 119, 115, 115, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130893955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130907885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130920655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130933485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130945995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130965144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"":"downstairs","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.130981224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.131006714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"0","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.131021424Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"1","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.131034254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"client":"2","":"downstairs","session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.131047224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.131059294Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"38301b99-015b-421e-84e5-dc4a887c4f17 is now active with session: e426a78d-a249-4f5a-9e39-af3b007b9764","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.131071874Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.131083703Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680,"session_id":"e426a78d-a249-4f5a-9e39-af3b007b9764"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T01:46:32.131095783Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680} {"msg":"Activated sub_volume 38301b99-015b-421e-84e5-dc4a887c4f17","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:32.131110453Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 420 len 28672 data: 11 8 7 7 8 9 12 03/30 Read block 156 len 36864 04/30 Read block 325 len 4096 05/30 Flush 06/30 Write block 543 len 12288 data: 12 10 11 07/30 Read block 183 len 28672 08/30 Write block 476 len 28672 data: 6 7 9 5 7 7 5 09/30 Read block 170 len 40960 10/30 Read block 2 len 20480 11/30 Flush 12/30 Write block 350 len 12288 data: 11 11 10 13/30 Write block 82 len 4096 data: 12 14/30 Read block 105 len 24576 15/30 Read block 5 len 16384 16/30 Read block 2 len 32768 17/30 Read block 91 len 8192 18/30 Read block 514 len 36864 19/30 Write block 343 len 24576 data: 7 7 9 8 7 8 20/30 Read block 84 len 8192 21/30 Read block 294 len 8192 22/30 Flush 23/30 Flush 24/30 Flush 25/30 Write block 185 len 12288 data: 4 4 5 26/30 Write block 262 len 8192 data: 7 7 27/30 Read block 55 len 24576 28/30 Read block 146 len 20480 29/30 Read block 583 len 36864 30/30 Write block 532 len 32768 data: 8 8 8 11 9 8 10 11 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:32.150784645Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1680} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 212 212 212 F F F 1 020-039 121 121 121 216 216 216 F F F 2 040-059 121 121 121 216 216 216 F F F 3 060-079 117 117 117 209 209 209 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 121 121 121 216 216 216 F F F 6 120-139 117 117 117 209 209 209 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 115 115 115 206 206 206 T T T 10 200-219 119 119 119 213 213 213 F F F 11 220-239 121 121 121 216 216 216 F F F 12 240-259 120 120 120 214 214 214 F F F 13 260-279 113 113 113 204 204 204 T T T 14 280-299 121 121 121 216 216 216 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 122 122 122 220 220 220 F F F 18 360-379 120 120 120 214 214 214 F F F 19 380-399 118 118 118 211 211 211 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 122 122 122 218 218 218 F F F 22 440-459 121 121 121 216 216 216 F F F 23 460-479 122 122 122 219 219 219 F F F 24 480-499 122 122 122 219 219 219 F F F 25 500-519 120 120 120 215 215 215 F F F 26 520-539 111 111 111 201 201 201 T T T 27 540-559 122 122 122 219 219 219 F F F 28 560-579 120 120 120 214 214 214 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 122, Max flush: 220 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 212 212 212 F F F 1 020-039 121 121 121 216 216 216 F F F 2 040-059 121 121 121 216 216 216 F F F 3 060-079 117 117 117 209 209 209 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 121 121 121 216 216 216 F F F 6 120-139 117 117 117 209 209 209 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 123 123 123 221 221 221 F F F 10 200-219 119 119 119 213 213 213 F F F 11 220-239 121 121 121 216 216 216 F F F 12 240-259 120 120 120 214 214 214 F F F 13 260-279 123 123 123 221 221 221 F F F 14 280-299 121 121 121 216 216 216 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 122 122 122 220 220 220 F F F 18 360-379 120 120 120 214 214 214 F F F 19 380-399 118 118 118 211 211 211 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 122 122 122 218 218 218 F F F 22 440-459 121 121 121 216 216 216 F F F 23 460-479 122 122 122 219 219 219 F F F 24 480-499 122 122 122 219 219 219 F F F 25 500-519 120 120 120 215 215 215 F F F 26 520-539 123 123 123 221 221 221 F F F 27 540-559 122 122 122 219 219 219 F F F 28 560-579 120 120 120 214 214 214 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 123, Max flush: 221 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.38321536Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.429209754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.429841139Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.429886108Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.429901488Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.429916048Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688} {"msg":"Crucible 4f1a55aa-e0ba-4502-8757-efde31e5e46c has session id: 91beb41a-ecc1-4da7-acec-1f09eb503496","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.430545392Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Upstairs opts: Upstairs UUID: 4f1a55aa-e0ba-4502-8757-efde31e5e46c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.430583822Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Crucible stats registered with UUID: 4f1a55aa-e0ba-4502-8757-efde31e5e46c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.430600392Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:36.430613081Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431043768Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431068137Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688} {"msg":"4f1a55aa-e0ba-4502-8757-efde31e5e46c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431133347Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431155356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"0","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431183776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"0","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431198016Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"1","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431212266Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"1","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431225126Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"2","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431238896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"2","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431251556Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431264955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431278215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"ds_connection connected from Ok(127.0.0.1:50770)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431413294Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"ds_connection connected from Ok(127.0.0.1:42256)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431451564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"ds_connection connected from Ok(127.0.0.1:47610)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.431478294Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.432101848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"0","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.432143248Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"1","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.432177847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"2","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.432641593Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"1","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.432767332Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"0","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433076779Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"2","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433283947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433386796Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[0]R flush_numbers[0..12]: [212, 216, 216, 209, 220, 216, 209, 213, 206, 221, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433627534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[0]R generation[0..12]: [118, 121, 121, 117, 122, 121, 117, 119, 115, 123, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433649044Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433663394Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[1]R flush_numbers[0..12]: [212, 216, 216, 209, 220, 216, 209, 213, 206, 221, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433677014Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[1]R generation[0..12]: [118, 121, 121, 117, 122, 121, 117, 119, 115, 123, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433691354Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433705073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[2]R flush_numbers[0..12]: [212, 216, 216, 209, 220, 216, 209, 213, 206, 221, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433718473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[2]R generation[0..12]: [118, 121, 121, 117, 122, 121, 117, 119, 115, 123, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433731723Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433748623Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433761613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433775803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433788393Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433800803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433819863Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433831512Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"0","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433844552Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"1","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433857682Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"client":"2","":"downstairs","session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433870962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433882952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"4f1a55aa-e0ba-4502-8757-efde31e5e46c is now active with session: 91beb41a-ecc1-4da7-acec-1f09eb503496","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433895542Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433907162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688,"session_id":"91beb41a-ecc1-4da7-acec-1f09eb503496"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433919062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688} {"msg":"Activated sub_volume 4f1a55aa-e0ba-4502-8757-efde31e5e46c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:36.433929902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 23 len 36864 data: 6 7 8 11 13 15 12 11 9 02/30 Read block 82 len 40960 03/30 Read block 390 len 28672 04/30 Write block 114 len 4096 data: 8 05/30 Write block 63 len 32768 data: 12 11 14 16 13 14 13 9 06/30 Read block 137 len 4096 07/30 Read block 15 len 28672 08/30 Flush 09/30 Write block 11 len 28672 data: 12 11 12 9 12 13 14 10/30 Read block 503 len 8192 11/30 Read block 251 len 20480 12/30 Write block 373 len 32768 data: 13 14 14 12 13 13 11 8 13/30 Read block 48 len 12288 14/30 Read block 297 len 32768 15/30 Read block 204 len 20480 16/30 Read block 68 len 16384 17/30 Read block 500 len 8192 18/30 Read block 39 len 24576 19/30 Write block 345 len 36864 data: 10 9 8 9 8 12 12 11 10 20/30 Flush 21/30 Write block 251 len 28672 data: 9 12 13 13 12 12 11 22/30 Write block 266 len 8192 data: 9 11 23/30 Read block 320 len 4096 24/30 Read block 413 len 24576 25/30 Read block 403 len 16384 26/30 Write block 251 len 20480 data: 10 13 14 14 13 27/30 Write block 137 len 32768 data: 10 8 7 7 7 7 6 8 28/30 Write block 501 len 4096 data: 11 29/30 Read block 62 len 8192 30/30 Write block 493 len 28672 data: 10 10 9 6 9 9 8 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:36.45512807Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1688} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 124 124 124 222 222 222 F F F 2 040-059 121 121 121 216 216 216 F F F 3 060-079 124 124 124 222 222 222 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 124 124 124 222 222 222 F F F 6 120-139 117 117 117 209 209 209 T T T 7 140-159 119 119 119 213 213 213 T T T 8 160-179 115 115 115 206 206 206 F F F 9 180-199 123 123 123 221 221 221 F F F 10 200-219 119 119 119 213 213 213 F F F 11 220-239 121 121 121 216 216 216 F F F 12 240-259 120 120 120 214 214 214 T T T 13 260-279 123 123 123 221 221 221 T T T 14 280-299 121 121 121 216 216 216 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 124 124 124 223 223 223 F F F 18 360-379 124 124 124 223 223 223 F F F 19 380-399 124 124 124 223 223 223 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 122 122 122 218 218 218 F F F 22 440-459 121 121 121 216 216 216 F F F 23 460-479 122 122 122 219 219 219 F F F 24 480-499 122 122 122 219 219 219 T T T 25 500-519 120 120 120 215 215 215 T T T 26 520-539 123 123 123 221 221 221 F F F 27 540-559 122 122 122 219 219 219 F F F 28 560-579 120 120 120 214 214 214 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 124, Max flush: 223 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 124 124 124 222 222 222 F F F 2 040-059 121 121 121 216 216 216 F F F 3 060-079 124 124 124 222 222 222 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 124 124 124 222 222 222 F F F 6 120-139 125 125 125 224 224 224 F F F 7 140-159 125 125 125 224 224 224 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 123 123 123 221 221 221 F F F 10 200-219 119 119 119 213 213 213 F F F 11 220-239 121 121 121 216 216 216 F F F 12 240-259 125 125 125 224 224 224 F F F 13 260-279 125 125 125 224 224 224 F F F 14 280-299 121 121 121 216 216 216 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 124 124 124 223 223 223 F F F 18 360-379 124 124 124 223 223 223 F F F 19 380-399 124 124 124 223 223 223 F F F 20 400-419 117 117 117 209 209 209 F F F 21 420-439 122 122 122 218 218 218 F F F 22 440-459 121 121 121 216 216 216 F F F 23 460-479 122 122 122 219 219 219 F F F 24 480-499 125 125 125 224 224 224 F F F 25 500-519 125 125 125 224 224 224 F F F 26 520-539 123 123 123 221 221 221 F F F 27 540-559 122 122 122 219 219 219 F F F 28 560-579 120 120 120 214 214 214 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 125, Max flush: 224 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.70469672Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:40.747019257Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.747375234Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.778287904Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.778917888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.778950558Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.778969677Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.778983947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696} {"msg":"Crucible 44f7e17e-d352-46d0-9d0f-ae72cf0538b2 has session id: 59019013-f49c-407d-a8b2-a6d9e2a947db","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.779533272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Upstairs opts: Upstairs UUID: 44f7e17e-d352-46d0-9d0f-ae72cf0538b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.779581452Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Crucible stats registered with UUID: 44f7e17e-d352-46d0-9d0f-ae72cf0538b2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.779602372Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:40.779618652Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780041578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780071898Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696} {"msg":"44f7e17e-d352-46d0-9d0f-ae72cf0538b2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780121627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780140117Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"0","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780174537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"0","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780191417Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"1","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780209356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"1","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780225466Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"2","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780242616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"2","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780264726Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780286676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780318425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"ds_connection connected from Ok(127.0.0.1:37309)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780351765Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"ds_connection connected from Ok(127.0.0.1:36507)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780379625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"ds_connection connected from Ok(127.0.0.1:50178)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.780593853Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.781155518Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"1","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.781188628Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"2","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.781429405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"0","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.78195561Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"1","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.78198576Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"2","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782085379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"0","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782638044Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782672544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[0]R flush_numbers[0..12]: [223, 222, 216, 222, 220, 222, 224, 224, 206, 221, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782692244Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[0]R generation[0..12]: [124, 124, 121, 124, 122, 124, 125, 125, 115, 123, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782711234Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782725694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[1]R flush_numbers[0..12]: [223, 222, 216, 222, 220, 222, 224, 224, 206, 221, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782739223Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[1]R generation[0..12]: [124, 124, 121, 124, 122, 124, 125, 125, 115, 123, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782752453Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782765913Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[2]R flush_numbers[0..12]: [223, 222, 216, 222, 220, 222, 224, 224, 206, 221, 213, 216]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782779133Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[2]R generation[0..12]: [124, 124, 121, 124, 122, 124, 125, 125, 115, 123, 119, 121]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782792473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782805883Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782818503Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782840373Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Next flush: 225","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:46:40.782864222Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782887782Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"":"downstairs","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782910162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782930292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"0","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782957952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"1","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.782981521Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"client":"2","":"downstairs","session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.783001641Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.783014201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"44f7e17e-d352-46d0-9d0f-ae72cf0538b2 is now active with session: 59019013-f49c-407d-a8b2-a6d9e2a947db","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.783026821Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.783038631Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696,"session_id":"59019013-f49c-407d-a8b2-a6d9e2a947db"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.78305054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696} {"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 msg":"Activated sub_volume 44f7e17e-d352-46d0-9d0f-ae72cf0538b2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:40.78306548Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 129 len 32768 data: 9 6 6 5 8 9 8 7 02/30 Read block 119 len 8192 03/30 Write block 216 len 32768 data: 12 12 12 12 14 15 14 13 04/30 Write block 221 len 20480 data: 16 15 14 11 8 05/30 Write block 401 len 12288 data: 14 15 15 06/30 Write block 38 len 36864 data: 8 8 9 10 10 13 12 9 8 07/30 Write block 544 len 36864 data: 11 12 10 10 11 12 10 13 12 08/30 Read block 535 len 4096 09/30 Read block 226 len 8192 10/30 Read block 454 len 32768 11/30 Write block 528 len 28672 data: 8 10 9 7 9 9 9 12/30 Read block 146 len 12288 13/30 Write block 66 len 40960 data: 17 14 15 14 10 7 7 8 8 9 14/30 Write block 273 len 32768 data: 11 10 10 10 12 15 15 16 15/30 Write block 513 len 12288 data: 14 14 12 16/30 Write block 492 len 24576 data: 8 11 11 10 7 10 17/30 Read block 400 len 4096 18/30 Read block 138 len 16384 19/30 Write block 32 len 4096 data: 10 20/30 Read block 301 len 16384 21/30 Write block 131 len 8192 data: 7 6 22/30 Write block 384 len 8192 data: 9 10 23/30 Read block 410 len 20480 24/30 Write block 428 len 8192 data: 10 11 25/30 Read block 84 len 20480 26/30 Write block 485 len 40960 data: 8 9 7 8 12 8 8 9 12 12 27/30 Read block 313 len 20480 28/30 Read block 5 len 24576 29/30 Read block 99 len 36864 30/30 Read block 522 len 4096 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:40.802297237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1696} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 124 124 124 222 222 222 T T T 2 040-059 121 121 121 216 216 216 T T T 3 060-079 124 124 124 222 222 222 T T T 4 080-099 122 122 122 220 220 220 F F F 5 100-119 124 124 124 222 222 222 F F F 6 120-139 125 125 125 224 224 224 T T T 7 140-159 125 125 125 224 224 224 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 123 123 123 221 221 221 F F F 10 200-219 119 119 119 213 213 213 T T T 11 220-239 121 121 121 216 216 216 T T T 12 240-259 125 125 125 224 224 224 F F F 13 260-279 125 125 125 224 224 224 T T T 14 280-299 121 121 121 216 216 216 T T T 15 300-319 118 118 118 210 210 210 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 124 124 124 223 223 223 F F F 18 360-379 124 124 124 223 223 223 F F F 19 380-399 124 124 124 223 223 223 T T T 20 400-419 117 117 117 209 209 209 T T T 21 420-439 122 122 122 218 218 218 T T T 22 440-459 121 121 121 216 216 216 F F F 23 460-479 122 122 122 219 219 219 F F F 24 480-499 125 125 125 224 224 224 T T T 25 500-519 125 125 125 224 224 224 T T T 26 520-539 123 123 123 221 221 221 T T T 27 540-559 122 122 122 219 219 219 T T T 28 560-579 120 120 120 214 214 214 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 125, Max flush: 224 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 126 126 126 225 225 225 F F F 2 040-059 126 126 126 225 225 225 F F F 3 060-079 126 126 126 225 225 225 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 124 124 124 222 222 222 F F F 6 120-139 126 126 126 225 225 225 F F F 7 140-159 125 125 125 224 224 224 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 123 123 123 221 221 221 F F F 10 200-219 126 126 126 225 225 225 F F F 11 220-239 126 126 126 225 225 225 F F F 12 240-259 125 125 125 224 224 224 F F F 13 260-279 126 126 126 225 225 225 F F F 14 280-299 126 126 126 225 225 225 F F F 15 300-319 118 118 118 210 210 210 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 124 124 124 223 223 223 F F F 18 360-379 124 124 124 223 223 223 F F F 19 380-399 126 126 126 225 225 225 F F F 20 400-419 126 126 126 225 225 225 F F F 21 420-439 126 126 126 225 225 225 F F F 22 440-459 121 121 121 216 216 216 F F F 23 460-479 122 122 122 219 219 219 F F F 24 480-499 126 126 126 225 225 225 F F F 25 500-519 126 126 126 225 225 225 F F F 26 520-539 126 126 126 225 225 225 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 120 120 120 214 214 214 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 126, Max flush: 225 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.127434812Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.171803141Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.172701703Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.172748902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.172765152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.172779842Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704} {"msg":"Crucible e1b39649-2500-44bc-8b3c-40748853cf10 has session id: 461b4f90-e550-4a20-bb28-b9853e1f0eca","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.173356127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Upstairs opts: Upstairs UUID: e1b39649-2500-44bc-8b3c-40748853cf10, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.173512196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Crucible stats registered with UUID: e1b39649-2500-44bc-8b3c-40748853cf10","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.173550635Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:45.173572665Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.17409497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.17412852Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704} {"msg":"e1b39649-2500-44bc-8b3c-40748853cf10 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.17417623Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.1741929Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"0","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174222309Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"0","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174236719Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"1","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174251139Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"1","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174264039Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"2","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174278449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"2","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174291208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174304528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174333448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"ds_connection connected from Ok(127.0.0.1:58968)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174377458Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"ds_connection connected from Ok(127.0.0.1:35513)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174408747Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"ds_connection connected from Ok(127.0.0.1:55098)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.174435877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.175155771Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"1","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.17518526Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"0","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.17520327Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"2","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.175750255Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"1","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.175892574Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"0","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.175926474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"2","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.17636061Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176446179Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[0]R flush_numbers[0..12]: [223, 225, 225, 225, 220, 222, 225, 224, 206, 221, 225, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176568048Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[0]R generation[0..12]: [124, 126, 126, 126, 122, 124, 126, 125, 115, 123, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176587938Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176602378Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[1]R flush_numbers[0..12]: [223, 225, 225, 225, 220, 222, 225, 224, 206, 221, 225, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176619038Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[1]R generation[0..12]: [124, 126, 126, 126, 122, 124, 126, 125, 115, 123, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176648127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176677037Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[2]R flush_numbers[0..12]: [223, 225, 225, 225, 220, 222, 225, 224, 206, 221, 225, 225]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176702707Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[2]R generation[0..12]: [124, 126, 126, 126, 122, 124, 126, 125, 115, 123, 126, 126]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176727657Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176742966Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Max found gen is 127","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176758926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Generation requested: 128 >= found:127","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:46:45.176782546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176809316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176833825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"":"downstairs","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176848836Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176860515Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"0","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176875615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"1","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176898505Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"client":"2","":"downstairs","session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176923175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176946144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"e1b39649-2500-44bc-8b3c-40748853cf10 is now active with session: 461b4f90-e550-4a20-bb28-b9853e1f0eca","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176964064Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.176980564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704,"session_id":"461b4f90-e550-4a20-bb28-b9853e1f0eca"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.177001404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704} {"msg":"Activated sub_volume e1b39649-2500-44bc-8b3c-40748853cf10","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:45.177020564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 433 len 40960 data: 9 8 10 9 9 7 8 8 7 8 02/30 Write block 447 len 4096 data: 12 03/30 Flush 04/30 Write block 502 len 32768 data: 10 11 10 9 9 12 11 11 05/30 Read block 280 len 16384 06/30 Read block 502 len 28672 07/30 Write block 396 len 28672 data: 17 20 15 15 15 15 16 08/30 Read block 73 len 4096 09/30 Read block 462 len 36864 10/30 Flush 11/30 Write block 365 len 4096 data: 4 12/30 Write block 215 len 24576 data: 9 13 13 13 13 15 13/30 Write block 477 len 20480 data: 8 10 6 8 8 14/30 Flush 15/30 Write block 284 len 12288 data: 15 9 10 16/30 Read block 366 len 24576 17/30 Read block 207 len 20480 18/30 Flush 19/30 Write block 489 len 28672 data: 13 9 9 10 13 13 11 20/30 Read block 385 len 28672 21/30 Flush 22/30 Write block 567 len 20480 data: 10 8 10 8 9 23/30 Read block 91 len 28672 24/30 Write block 25 len 40960 data: 9 12 14 16 13 12 10 11 9 8 25/30 Read block 230 len 32768 26/30 Flush 27/30 Write block 311 len 20480 data: 10 9 9 7 6 28/30 Read block 458 len 20480 29/30 Read block 294 len 16384 30/30 Write block 186 len 16384 data: 5 6 4 6 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:45.198194242Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1704} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 126 126 126 225 225 225 F F F 3 060-079 126 126 126 225 225 225 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 124 124 124 222 222 222 F F F 6 120-139 126 126 126 225 225 225 F F F 7 140-159 125 125 125 224 224 224 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 123 123 123 221 221 221 T T T 10 200-219 128 128 128 228 228 228 F F F 11 220-239 128 128 128 228 228 228 F F F 12 240-259 125 125 125 224 224 224 F F F 13 260-279 126 126 126 225 225 225 F F F 14 280-299 128 128 128 229 229 229 F F F 15 300-319 118 118 118 210 210 210 T T T 16 320-339 121 121 121 216 216 216 F F F 17 340-359 124 124 124 223 223 223 F F F 18 360-379 128 128 128 228 228 228 F F F 19 380-399 128 128 128 227 227 227 F F F 20 400-419 128 128 128 227 227 227 F F F 21 420-439 128 128 128 226 226 226 F F F 22 440-459 128 128 128 226 226 226 F F F 23 460-479 128 128 128 228 228 228 F F F 24 480-499 128 128 128 230 230 230 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 126 126 126 225 225 225 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 128 128 128 231 231 231 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 128, Max flush: 231 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 126 126 126 225 225 225 F F F 3 060-079 126 126 126 225 225 225 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 124 124 124 222 222 222 F F F 6 120-139 126 126 126 225 225 225 F F F 7 140-159 125 125 125 224 224 224 F F F 8 160-179 115 115 115 206 206 206 F F F 9 180-199 129 129 129 232 232 232 F F F 10 200-219 128 128 128 228 228 228 F F F 11 220-239 128 128 128 228 228 228 F F F 12 240-259 125 125 125 224 224 224 F F F 13 260-279 126 126 126 225 225 225 F F F 14 280-299 128 128 128 229 229 229 F F F 15 300-319 129 129 129 232 232 232 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 124 124 124 223 223 223 F F F 18 360-379 128 128 128 228 228 228 F F F 19 380-399 128 128 128 227 227 227 F F F 20 400-419 128 128 128 227 227 227 F F F 21 420-439 128 128 128 226 226 226 F F F 22 440-459 128 128 128 226 226 226 F F F 23 460-479 128 128 128 228 228 228 F F F 24 480-499 128 128 128 230 230 230 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 126 126 126 225 225 225 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 128 128 128 231 231 231 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 129, Max flush: 232 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.420042763Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.463885776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.464507121Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46455932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46458403Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.4646023Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712} {"msg":"Crucible 8492060f-4a93-4ca3-b18f-0a2b70bcc31e has session id: 2e8af94c-ec66-4317-81b5-c4ef7f0e03df","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465181195Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Upstairs opts: Upstairs UUID: 8492060f-4a93-4ca3-b18f-0a2b70bcc31e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465219194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Crucible stats registered with UUID: 8492060f-4a93-4ca3-b18f-0a2b70bcc31e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465245264Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:49.465267654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46569706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46573034Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712} {"msg":"8492060f-4a93-4ca3-b18f-0a2b70bcc31e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465787899Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465814479Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"0","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465845169Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"0","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465859878Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"1","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465875378Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"1","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465897858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"2","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465926718Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"2","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465949568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465973277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.465998937Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"ds_connection connected from Ok(127.0.0.1:42013)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.466025827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"ds_connection connected from Ok(127.0.0.1:45784)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.466070167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"ds_connection connected from Ok(127.0.0.1:62029)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.466098536Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.466556142Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"2","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.466592662Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"0","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.467123167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"0","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.467243896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"2","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.467663002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"1","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.467703832Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.467729742Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.468303927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"1","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[0]R flush_numbers[0..12]: [223, 231, 225, 225, 220, 222, 225, 224, 206, 232, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.468896451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[0]R generation[0..12]: [124, 128, 126, 126, 122, 124, 126, 125, 115, 129, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.468918571Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.468933451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[1]R flush_numbers[0..12]: [223, 231, 225, 225, 220, 222, 225, 224, 206, 232, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.468947331Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[1]R generation[0..12]: [124, 128, 126, 126, 122, 124, 126, 125, 115, 129, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46896069Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46897434Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[2]R flush_numbers[0..12]: [223, 231, 225, 225, 220, 222, 225, 224, 206, 232, 228, 228]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46898788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[2]R generation[0..12]: [124, 128, 126, 126, 122, 124, 126, 125, 115, 129, 128, 128]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46900166Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46901548Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46902808Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46904089Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46905315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.46906554Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"":"downstairs","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469081309Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-03-18T01:46:49.469102119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"0","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469115499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"1","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469128439Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"client":"2","":"downstairs","session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469141499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469153419Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"8492060f-4a93-4ca3-b18f-0a2b70bcc31e is now active with session: 2e8af94c-ec66-4317-81b5-c4ef7f0e03df","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469165989Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469177679Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712,"session_id":"2e8af94c-ec66-4317-81b5-c4ef7f0e03df"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469189728Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712} {"msg":"Activated sub_volume 8492060f-4a93-4ca3-b18f-0a2b70bcc31e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:46:49.469200598Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 167 len 28672 data: 4 6 6 6 5 6 8 02/30 Read block 367 len 28672 03/30 Write block 280 len 12288 data: 17 17 18 04/30 Read block 334 len 40960 05/30 Read block 125 len 4096 06/30 Read block 529 len 8192 07/30 Write block 385 len 32768 data: 11 10 12 15 15 17 15 16 08/30 Read block 215 len 4096 09/30 Read block 197 len 24576 10/30 Read block 482 len 24576 11/30 Write block 239 len 16384 data: 10 14 11 10 12/30 Write block 132 len 24576 data: 7 9 10 9 8 11 13/30 Read block 148 len 28672 14/30 Write block 351 len 40960 data: 13 12 11 5 7 8 10 10 10 10 15/30 Read block 84 len 40960 16/30 Read block 442 len 36864 17/30 Write block 116 len 32768 data: 6 9 9 10 15 15 15 14 18/30 Write block 475 len 16384 data: 6 7 9 11 19/30 Read block 255 len 16384 20/30 Read block 416 len 16384 21/30 Write block 562 len 32768 data: 5 7 8 11 9 11 9 11 22/30 Read block 98 len 32768 23/30 Write block 571 len 36864 data: 10 9 7 9 10 9 13 15 18 24/30 Read block 54 len 16384 25/30 Read block 581 len 8192 26/30 Read block 168 len 4096 27/30 Write block 350 len 20480 data: 13 14 13 12 6 28/30 Write block 132 len 40960 data: 8 10 11 10 9 12 9 8 8 8 29/30 Read block 538 len 20480 30/30 Write block 568 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:46:49.491285048Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 126 126 126 225 225 225 F F F 3 060-079 126 126 126 225 225 225 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 124 124 124 222 222 222 T T T 6 120-139 126 126 126 225 225 225 T T T 7 140-159 125 125 125 224 224 224 T T T 8 160-179 115 115 115 206 206 206 T T T 9 180-199 129 129 129 232 232 232 F F F 10 200-219 128 128 128 228 228 228 F F F 11 220-239 128 128 128 228 228 228 T T T 12 240-259 125 125 125 224 224 224 T T T 13 260-279 126 126 126 225 225 225 F F F 14 280-299 128 128 128 229 229 229 T T T 15 300-319 129 129 129 232 232 232 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 124 124 124 223 223 223 T T T 18 360-379 128 128 128 228 228 228 T T T 19 380-399 128 128 128 227 227 227 T T T 20 400-419 128 128 128 227 227 227 F F F 21 420-439 128 128 128 226 226 226 F F F 22 440-459 128 128 128 226 226 226 F F F 23 460-479 128 128 128 228 228 228 T T T 24 480-499 128 128 128 230 230 230 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 126 126 126 225 225 225 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 128 128 128 231 231 231 T T T 29 580-599 119 119 119 213 213 213 F F F Max gen: 129, Max flush: 232 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 126 126 126 225 225 225 F F F 3 060-079 126 126 126 225 225 225 F F F 4 080-099 122 122 122 220 220 220 F F F 5 100-119 130 130 130 233 233 233 F F F 6 120-139 130 130 130 233 233 233 F F F 7 140-159 130 130 130 233 233 233 F F F 8 160-179 130 130 130 233 233 233 F F F 9 180-199 129 129 129 232 232 232 F F F 10 200-219 128 128 128 228 228 228 F F F 11 220-239 130 130 130 233 233 233 F F F 12 240-259 130 130 130 233 233 233 F F F 13 260-279 126 126 126 225 225 225 F F F 14 280-299 130 130 130 233 233 233 F F F 15 300-319 129 129 129 232 232 232 F F F 16 320-339 121 121 121 216 216 216 F F F 17 340-359 130 130 130 233 233 233 F F F 18 360-379 130 130 130 233 233 233 F F F 19 380-399 130 130 130 233 233 233 F F F 20 400-419 128 128 128 227 227 227 F F F 21 420-439 128 128 128 226 226 226 F F F 22 440-459 128 128 128 226 226 226 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 128 128 128 230 230 230 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 126 126 126 225 225 225 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 130 130 130 233 233 233 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 130, Max flush: 233 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.767618487Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.813041175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.813690929Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.813734339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.813750059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.813764619Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720} {"msg":"Crucible 19e3796b-86bd-44e5-9bbf-bcfdadeb8287 has session id: daf2a278-88ea-4b3b-a973-504cbd548a30","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.814393493Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Upstairs opts: Upstairs UUID: 19e3796b-86bd-44e5-9bbf-bcfdadeb8287, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.814507592Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Crucible stats registered with UUID: 19e3796b-86bd-44e5-9bbf-bcfdadeb8287","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.814544952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:03.814569091Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815117646Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815154576Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720} {"msg":"19e3796b-86bd-44e5-9bbf-bcfdadeb8287 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815270285Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815292985Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"0","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815327544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"0","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815342115Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"1","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815356694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"1","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815370914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"2","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815385654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"2","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815405624Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815431414Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815512303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"ds_connection connected from Ok(127.0.0.1:57139)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815617772Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"ds_connection connected from Ok(127.0.0.1:50751)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815653482Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"ds_connection connected from Ok(127.0.0.1:55475)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.815690261Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.816473084Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"2","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.816512714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"0","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.817091908Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"0","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.817225027Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"1","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.817257577Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"2","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.817710613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.817868232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.8180568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"1","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[0]R flush_numbers[0..12]: [223, 231, 225, 225, 220, 233, 233, 233, 233, 232, 228, 233]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818612965Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[0]R generation[0..12]: [124, 128, 126, 126, 122, 130, 130, 130, 130, 129, 128, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818637795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818654654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[1]R flush_numbers[0..12]: [223, 231, 225, 225, 220, 233, 233, 233, 233, 232, 228, 233]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818672384Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[1]R generation[0..12]: [124, 128, 126, 126, 122, 130, 130, 130, 130, 129, 128, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818686494Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818700524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[2]R flush_numbers[0..12]: [223, 231, 225, 225, 220, 233, 233, 233, 233, 232, 228, 233]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818714524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[2]R generation[0..12]: [124, 128, 126, 126, 122, 130, 130, 130, 130, 129, 128, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818727764Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818741484Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Max found gen is 131","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818754173Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Generation requested: 132 >= found:131","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818767033Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Next flush: 234","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:47:03.818779763Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818796253Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"":"downstairs","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818809033Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818820713Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"0","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818833723Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"1","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818846643Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"client":"2","":"downstairs","session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818859623Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818873272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"19e3796b-86bd-44e5-9bbf-bcfdadeb8287 is now active with session: daf2a278-88ea-4b3b-a973-504cbd548a30","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818886062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818897862Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720,"session_id":"daf2a278-88ea-4b3b-a973-504cbd548a30"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818909942Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720} {"msg":"Activated sub_volume 19e3796b-86bd-44e5-9bbf-bcfdadeb8287","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:03.818921062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 4096 data: 11 02/30 Write block 94 len 32768 data: 10 9 7 6 8 8 10 11 03/30 Read block 483 len 8192 04/30 Read block 148 len 8192 05/30 Write block 312 len 36864 data: 10 10 8 7 10 10 12 11 11 06/30 Flush 07/30 Write block 290 len 4096 data: 14 08/30 Read block 131 len 12288 09/30 Flush 10/30 Write block 382 len 40960 data: 9 10 10 12 11 13 16 16 18 16 11/30 Write block 97 len 16384 data: 7 9 9 11 12/30 Read block 592 len 16384 13/30 Flush 14/30 Write block 521 len 40960 data: 13 10 9 11 11 12 12 9 11 10 15/30 Write block 141 len 32768 data: 9 8 7 9 8 8 8 9 16/30 Write block 51 len 12288 data: 10 13 10 17/30 Read block 240 len 16384 18/30 Read block 487 len 32768 19/30 Read block 204 len 8192 20/30 Write block 176 len 16384 data: 7 6 6 8 21/30 Write block 269 len 40960 data: 9 10 10 9 12 11 11 11 13 16 22/30 Write block 373 len 36864 data: 14 15 15 13 14 14 12 9 7 23/30 Write block 69 len 20480 data: 15 11 8 8 9 24/30 Write block 526 len 40960 data: 13 13 10 12 11 8 10 10 10 12 25/30 Write block 221 len 16384 data: 17 16 15 12 26/30 Write block 147 len 32768 data: 9 10 9 9 9 9 9 9 27/30 Read block 499 len 20480 28/30 Write block 406 len 28672 data: 9 6 6 6 7 10 10 29/30 Read block 267 len 36864 30/30 Write block 200 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 9 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 16 15 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:03.838413626Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1720} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 126 126 126 225 225 225 T T T 3 060-079 126 126 126 225 225 225 T T T 4 080-099 132 132 132 236 236 236 F F F 5 100-119 132 132 132 236 236 236 F F F 6 120-139 130 130 130 233 233 233 F F F 7 140-159 130 130 130 233 233 233 T T T 8 160-179 130 130 130 233 233 233 T T T 9 180-199 129 129 129 232 232 232 F F F 10 200-219 128 128 128 228 228 228 T T T 11 220-239 130 130 130 233 233 233 T T T 12 240-259 132 132 132 234 234 234 F F F 13 260-279 126 126 126 225 225 225 T T T 14 280-299 132 132 132 235 235 235 F F F 15 300-319 132 132 132 234 234 234 F F F 16 320-339 132 132 132 234 234 234 F F F 17 340-359 130 130 130 233 233 233 F F F 18 360-379 130 130 130 233 233 233 T T T 19 380-399 132 132 132 236 236 236 T T T 20 400-419 128 128 128 227 227 227 T T T 21 420-439 128 128 128 226 226 226 F F F 22 440-459 128 128 128 226 226 226 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 128 128 128 230 230 230 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 126 126 126 225 225 225 T T T 27 540-559 126 126 126 225 225 225 F F F 28 560-579 130 130 130 233 233 233 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 132, Max flush: 236 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 133 133 133 237 237 237 F F F 4 080-099 132 132 132 236 236 236 F F F 5 100-119 132 132 132 236 236 236 F F F 6 120-139 130 130 130 233 233 233 F F F 7 140-159 133 133 133 237 237 237 F F F 8 160-179 133 133 133 237 237 237 F F F 9 180-199 129 129 129 232 232 232 F F F 10 200-219 133 133 133 237 237 237 F F F 11 220-239 133 133 133 237 237 237 F F F 12 240-259 132 132 132 234 234 234 F F F 13 260-279 133 133 133 237 237 237 F F F 14 280-299 132 132 132 235 235 235 F F F 15 300-319 132 132 132 234 234 234 F F F 16 320-339 132 132 132 234 234 234 F F F 17 340-359 130 130 130 233 233 233 F F F 18 360-379 133 133 133 237 237 237 F F F 19 380-399 133 133 133 237 237 237 F F F 20 400-419 133 133 133 237 237 237 F F F 21 420-439 128 128 128 226 226 226 F F F 22 440-459 128 128 128 226 226 226 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 128 128 128 230 230 230 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 133 133 133 237 237 237 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 130 130 130 233 233 233 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 133, Max flush: 237 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.117398929Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.163105046Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.16374237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.16379635Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.163814279Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.163832479Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728} {"msg":"Crucible 1964b59f-6889-4242-8982-3f5b552686a2 has session id: bfd061fa-6077-4ac4-bd49-6a496ecd35c4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.164475783Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Upstairs opts: Upstairs UUID: 1964b59f-6889-4242-8982-3f5b552686a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.164513043Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Crucible stats registered with UUID: 1964b59f-6889-4242-8982-3f5b552686a2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.164528743Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:18.164541363Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.164989449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165014268Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728} {"msg":"1964b59f-6889-4242-8982-3f5b552686a2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165079648Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165097228Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"0","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165130158Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"0","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165152597Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"1","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165172667Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"1","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165186057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"2","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165200247Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"2","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165213647Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165227277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165240557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:47347)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165254096Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:39719)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165408715Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63634)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.165447824Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.166101179Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"0","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.166138208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"1","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.166821682Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"0","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.166872992Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"1","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.167572525Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.167607875Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.168063221Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"2","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.168635616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"2","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[0]R flush_numbers[0..12]: [223, 231, 237, 237, 236, 236, 233, 237, 237, 232, 237, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169197101Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[0]R generation[0..12]: [124, 128, 133, 133, 132, 132, 130, 133, 133, 129, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.16922838Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.16924837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[1]R flush_numbers[0..12]: [223, 231, 237, 237, 236, 236, 233, 237, 237, 232, 237, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.16926627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[1]R generation[0..12]: [124, 128, 133, 133, 132, 132, 130, 133, 133, 129, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.16928318Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.16930021Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[2]R flush_numbers[0..12]: [223, 231, 237, 237, 236, 236, 233, 237, 237, 232, 237, 237]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.1693169Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[2]R generation[0..12]: [124, 128, 133, 133, 132, 132, 130, 133, 133, 129, 133, 133]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169333749Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169351179Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169382249Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169400129Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169416109Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169431928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"":"downstairs","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169447858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169462688Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"0","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169478918Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"1","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169494948Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"client":"2","":"downstairs","session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169511148Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169526028Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"1964b59f-6889-4242-8982-3f5b552686a2 is now active with session: bfd061fa-6077-4ac4-bd49-6a496ecd35c4","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169541878Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169556407Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728,"session_id":"bfd061fa-6077-4ac4-bd49-6a496ecd35c4"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169571367Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728} {"msg":"Activated sub_volume 1964b59f-6889-4242-8982-3f5b552686a2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:18.169584877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 393 len 24576 data: 15 13 12 18 21 16 02/30 Flush 03/30 Write block 237 len 20480 data: 9 11 11 15 12 04/30 Read block 393 len 40960 05/30 Read block 288 len 40960 06/30 Write block 285 len 8192 data: 10 11 07/30 Read block 307 len 16384 08/30 Read block 525 len 40960 09/30 Read block 78 len 16384 10/30 Write block 301 len 28672 data: 9 12 10 9 7 7 8 11/30 Read block 65 len 4096 12/30 Read block 86 len 40960 13/30 Write block 439 len 36864 data: 9 9 8 9 8 12 11 11 13 14/30 Write block 259 len 24576 data: 7 7 5 8 8 12 15/30 Flush 16/30 Write block 75 len 4096 data: 10 17/30 Read block 257 len 4096 18/30 Read block 505 len 20480 19/30 Read block 378 len 20480 20/30 Write block 124 len 24576 data: 13 11 11 11 9 10 21/30 Flush 22/30 Write block 64 len 16384 data: 12 15 18 15 23/30 Flush 24/30 Write block 0 len 8192 data: 3 5 25/30 Write block 68 len 8192 data: 16 16 26/30 Read block 463 len 16384 27/30 Read block 257 len 28672 28/30 Read block 406 len 12288 29/30 Read block 477 len 16384 30/30 Write block 591 len 32768 data: 10 12 14 13 12 10 5 4 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:18.19138979Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1728} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 T T T 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 134 134 134 241 241 241 T T T 4 080-099 132 132 132 236 236 236 F F F 5 100-119 132 132 132 236 236 236 F F F 6 120-139 134 134 134 240 240 240 F F F 7 140-159 133 133 133 237 237 237 F F F 8 160-179 133 133 133 237 237 237 F F F 9 180-199 129 129 129 232 232 232 F F F 10 200-219 133 133 133 237 237 237 F F F 11 220-239 134 134 134 239 239 239 F F F 12 240-259 134 134 134 239 239 239 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 134 134 134 239 239 239 F F F 15 300-319 134 134 134 239 239 239 F F F 16 320-339 132 132 132 234 234 234 F F F 17 340-359 130 130 130 233 233 233 F F F 18 360-379 133 133 133 237 237 237 F F F 19 380-399 134 134 134 238 238 238 F F F 20 400-419 133 133 133 237 237 237 F F F 21 420-439 134 134 134 239 239 239 F F F 22 440-459 134 134 134 239 239 239 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 128 128 128 230 230 230 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 133 133 133 237 237 237 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 130 130 130 233 233 233 F F F 29 580-599 119 119 119 213 213 213 T T T Max gen: 134, Max flush: 241 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 242 242 242 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 135 135 135 242 242 242 F F F 4 080-099 132 132 132 236 236 236 F F F 5 100-119 132 132 132 236 236 236 F F F 6 120-139 134 134 134 240 240 240 F F F 7 140-159 133 133 133 237 237 237 F F F 8 160-179 133 133 133 237 237 237 F F F 9 180-199 129 129 129 232 232 232 F F F 10 200-219 133 133 133 237 237 237 F F F 11 220-239 134 134 134 239 239 239 F F F 12 240-259 134 134 134 239 239 239 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 134 134 134 239 239 239 F F F 15 300-319 134 134 134 239 239 239 F F F 16 320-339 132 132 132 234 234 234 F F F 17 340-359 130 130 130 233 233 233 F F F 18 360-379 133 133 133 237 237 237 F F F 19 380-399 134 134 134 238 238 238 F F F 20 400-419 133 133 133 237 237 237 F F F 21 420-439 134 134 134 239 239 239 F F F 22 440-459 134 134 134 239 239 239 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 128 128 128 230 230 230 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 133 133 133 237 237 237 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 130 130 130 233 233 233 F F F 29 580-599 135 135 135 242 242 242 F F F Max gen: 135, Max flush: 242 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.429220686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:22.472296726Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.472649353Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.50397862Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.504532105Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.504575014Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.504591984Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.504610264Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736} {"msg":"Crucible cd92da66-c9bc-421e-99c7-783a681d3428 has session id: cf76d579-7733-495c-b5ba-7abeec5558af","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505259408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Upstairs opts: Upstairs UUID: cd92da66-c9bc-421e-99c7-783a681d3428, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505298518Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Crucible stats registered with UUID: cd92da66-c9bc-421e-99c7-783a681d3428","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505318277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:22.505334867Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505713894Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505738694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736} {"msg":"cd92da66-c9bc-421e-99c7-783a681d3428 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505797123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505815173Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"0","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505844333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"0","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505858693Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"1","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505890092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"1","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505917102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"2","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505942262Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"2","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505966902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.505992011Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.506025031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"ds_connection connected from Ok(127.0.0.1:62749)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.506056071Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"ds_connection connected from Ok(127.0.0.1:63401)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.506085171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"ds_connection connected from Ok(127.0.0.1:41992)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.506305679Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.506774544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"1","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.506803764Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"2","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.506820044Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"0","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.507365159Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"1","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.507466188Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"0","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.507578217Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"2","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.507875685Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.507986533Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[0]R flush_numbers[0..12]: [242, 231, 237, 242, 236, 236, 240, 237, 237, 232, 237, 239]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508100962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[0]R generation[0..12]: [135, 128, 133, 135, 132, 132, 134, 133, 133, 129, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508121832Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508140152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[1]R flush_numbers[0..12]: [242, 231, 237, 242, 236, 236, 240, 237, 237, 232, 237, 239]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508155612Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[1]R generation[0..12]: [135, 128, 133, 135, 132, 132, 134, 133, 133, 129, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508169692Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508183522Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[2]R flush_numbers[0..12]: [242, 231, 237, 242, 236, 236, 240, 237, 237, 232, 237, 239]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508197082Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[2]R generation[0..12]: [135, 128, 133, 135, 132, 132, 134, 133, 133, 129, 133, 134]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508210591Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508224271Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508237001Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508249871Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508268511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508281161Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"":"downstairs","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508293981Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.508305621Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"0","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.50831846Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"1","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.5083332Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"client":"2","":"downstairs","session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.50834641Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.50835849Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"cd92da66-c9bc-421e-99c7-783a681d3428 is now active with session: cf76d579-7733-495c-b5ba-7abeec5558af","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.50837176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.50838339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736,"session_id":"cf76d579-7733-495c-b5ba-7abeec5558af"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:22.50839524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736} {"msg":"Activated sub_volume cd92da66-c9bc-421e-99c7-783a681d3428","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-18T01:47:22.50840617Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 361 len 12288 data: 7 6 6 02/30 Write block 419 len 16384 data: 11 12 9 8 03/30 Read block 168 len 32768 04/30 Read block 27 len 12288 05/30 Read block 149 len 32768 06/30 Read block 487 len 32768 07/30 Write block 200 len 12288 data: 10 7 8 08/30 Write block 118 len 12288 data: 10 11 16 09/30 Flush 10/30 Write block 213 len 40960 data: 10 11 10 14 14 14 14 16 18 17 11/30 Write block 524 len 8192 data: 12 12 12/30 Read block 95 len 16384 13/30 Read block 343 len 32768 14/30 Write block 241 len 24576 data: 13 11 9 8 8 7 15/30 Write block 70 len 36864 data: 12 9 9 10 9 11 10 11 12 16/30 Write block 387 len 24576 data: 14 17 17 19 17 17 17/30 Write block 490 len 12288 data: 10 10 11 18/30 Read block 332 len 16384 19/30 Write block 178 len 4096 data: 7 20/30 Read block 102 len 24576 21/30 Read block 290 len 4096 22/30 Read block 507 len 20480 23/30 Read block 325 len 40960 24/30 Write block 555 len 20480 data: 11 11 12 7 7 25/30 Read block 287 len 16384 26/30 Write block 185 len 40960 data: 5 6 7 5 7 6 6 7 6 7 27/30 Write block 558 len 4096 data: 8 28/30 Read block 142 len 20480 29/30 Write block 332 len 40960 data: 14 13 15 16 15 18 13 12 13 13 30/30 Read block 29 len 20480 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 10 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 21 19 19 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:22.528076292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1736} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 242 242 242 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 135 135 135 242 242 242 T T T 4 080-099 132 132 132 236 236 236 F F F 5 100-119 136 136 136 243 243 243 F F F 6 120-139 136 136 136 243 243 243 F F F 7 140-159 133 133 133 237 237 237 F F F 8 160-179 133 133 133 237 237 237 T T T 9 180-199 129 129 129 232 232 232 T T T 10 200-219 136 136 136 243 243 243 T T T 11 220-239 134 134 134 239 239 239 T T T 12 240-259 134 134 134 239 239 239 T T T 13 260-279 134 134 134 239 239 239 F F F 14 280-299 134 134 134 239 239 239 F F F 15 300-319 134 134 134 239 239 239 F F F 16 320-339 132 132 132 234 234 234 T T T 17 340-359 130 130 130 233 233 233 T T T 18 360-379 136 136 136 243 243 243 F F F 19 380-399 134 134 134 238 238 238 T T T 20 400-419 136 136 136 243 243 243 F F F 21 420-439 136 136 136 243 243 243 F F F 22 440-459 134 134 134 239 239 239 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 128 128 128 230 230 230 T T T 25 500-519 128 128 128 227 227 227 F F F 26 520-539 133 133 133 237 237 237 T T T 27 540-559 126 126 126 225 225 225 T T T 28 560-579 130 130 130 233 233 233 F F F 29 580-599 135 135 135 242 242 242 F F F Max gen: 136, Max flush: 243 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 242 242 242 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 137 137 137 244 244 244 F F F 4 080-099 132 132 132 236 236 236 F F F 5 100-119 136 136 136 243 243 243 F F F 6 120-139 136 136 136 243 243 243 F F F 7 140-159 133 133 133 237 237 237 F F F 8 160-179 137 137 137 244 244 244 F F F 9 180-199 137 137 137 244 244 244 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 137 137 137 244 244 244 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 134 134 134 239 239 239 F F F 15 300-319 134 134 134 239 239 239 F F F 16 320-339 137 137 137 244 244 244 F F F 17 340-359 137 137 137 244 244 244 F F F 18 360-379 136 136 136 243 243 243 F F F 19 380-399 137 137 137 244 244 244 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 136 136 136 243 243 243 F F F 22 440-459 134 134 134 239 239 239 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 128 128 128 227 227 227 F F F 26 520-539 137 137 137 244 244 244 F F F 27 540-559 137 137 137 244 244 244 F F F 28 560-579 130 130 130 233 233 233 F F F 29 580-599 135 135 135 242 242 242 F F F Max gen: 137, Max flush: 244 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.833680054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.878209212Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.878875656Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.878916395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.878930505Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.878944955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744} {"msg":"Crucible 64e403c4-8e92-4237-8128-6850770c09c0 has session id: 79c564c3-b325-49ef-a8c0-3d84f861b4f9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.87948526Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Upstairs opts: Upstairs UUID: 64e403c4-8e92-4237-8128-6850770c09c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.87952418Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Crucible stats registered with UUID: 64e403c4-8e92-4237-8128-6850770c09c0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.87954897Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:26.879569569Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880027315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880058875Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744} {"msg":"64e403c4-8e92-4237-8128-6850770c09c0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880117774Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880145744Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"0","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880184904Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"0","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880212044Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"1","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880238083Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"1","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880263473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"2","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880304163Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"2","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880329652Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880357872Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880382312Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41399)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880407442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41451)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880433231Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42807)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.880463561Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.881150935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"0","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.881184355Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"1","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.881313824Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"2","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.881784499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"0","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.881862569Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"1","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882122396Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"2","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882466833Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882502103Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[0]R flush_numbers[0..12]: [242, 231, 237, 244, 236, 243, 243, 237, 244, 244, 244, 244]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882644411Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[0]R generation[0..12]: [135, 128, 133, 137, 132, 136, 136, 133, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882667291Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882681981Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[1]R flush_numbers[0..12]: [242, 231, 237, 244, 236, 243, 243, 237, 244, 244, 244, 244]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882696221Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[1]R generation[0..12]: [135, 128, 133, 137, 132, 136, 136, 133, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882710121Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882724171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[2]R flush_numbers[0..12]: [242, 231, 237, 244, 236, 243, 243, 237, 244, 244, 244, 244]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882737621Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[2]R generation[0..12]: [135, 128, 133, 137, 132, 136, 136, 133, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882754481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882768301Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.88278114Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.88279444Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.88280723Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.88281983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"":"downstairs","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.88283252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.88284427Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"0","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.88285756Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"1","":Wait for a query_work_queue command to finish before sending IO "downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882879489Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"client":"2","":"downstairs","session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882892519Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882904739Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"64e403c4-8e92-4237-8128-6850770c09c0 is now active with session: 79c564c3-b325-49ef-a8c0-3d84f861b4f9","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882917329Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882929079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744,"session_id":"79c564c3-b325-49ef-a8c0-3d84f861b4f9"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882940969Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744} {"msg":"Activated sub_volume 64e403c4-8e92-4237-8128-6850770c09c0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:26.882951969Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 132 len 36864 data: 9 11 12 11 10 13 10 9 9 03/30 Write block 369 len 40960 data: 6 6 9 13 15 16 16 14 15 15 04/30 Read block 256 len 8192 05/30 Read block 147 len 40960 06/30 Read block 372 len 20480 07/30 Write block 127 len 40960 data: 12 10 11 7 8 10 12 13 12 11 08/30 Read block 361 len 16384 09/30 Read block 61 len 32768 10/30 Write block 71 len 40960 data: 10 10 11 10 12 11 12 13 11 12 11/30 Read block 123 len 24576 12/30 Write block 519 len 36864 data: 13 11 14 11 10 13 13 14 14 13/30 Write block 77 len 4096 data: 13 14/30 Read block 334 len 16384 15/30 Read block 17 len 8192 16/30 Read block 557 len 20480 17/30 Read block 26 len 8192 18/30 Read block 199 len 4096 19/30 Write block 361 len 20480 data: 8 7 7 6 5 20/30 Write block 500 len 40960 data: 11 12 11 12 11 10 10 13 12 12 21/30 Read block 566 len 32768 22/30 Write block 549 len 32768 data: 13 11 14 13 14 11 12 12 23/30 Write block 130 len 12288 data: 8 9 11 24/30 Write block 95 len 40960 data: 10 8 8 10 10 12 12 10 12 8 25/30 Flush 26/30 Write block 138 len 36864 data: 11 10 10 10 9 8 10 9 9 27/30 Read block 442 len 24576 28/30 Write block 593 len 20480 data: 15 14 13 11 6 29/30 Read block 213 len 24576 30/30 Read block 159 len 40960 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:26.905263217Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1744} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 242 242 242 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 138 138 138 246 246 246 F F F 5 100-119 138 138 138 246 246 246 F F F 6 120-139 138 138 138 246 246 246 T T T 7 140-159 138 138 138 246 246 246 T T T 8 160-179 137 137 137 244 244 244 F F F 9 180-199 137 137 137 244 244 244 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 137 137 137 244 244 244 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 134 134 134 239 239 239 F F F 15 300-319 134 134 134 239 239 239 F F F 16 320-339 137 137 137 244 244 244 F F F 17 340-359 137 137 137 244 244 244 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 137 137 137 244 244 244 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 136 136 136 243 243 243 F F F 22 440-459 134 134 134 239 239 239 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 138 138 138 246 246 246 F F F 26 520-539 138 138 138 246 246 246 F F F 27 540-559 138 138 138 246 246 246 F F F 28 560-579 130 130 130 233 233 233 F F F 29 580-599 135 135 135 242 242 242 T T T Max gen: 138, Max flush: 246 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 242 242 242 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 138 138 138 246 246 246 F F F 5 100-119 138 138 138 246 246 246 F F F 6 120-139 139 139 139 247 247 247 F F F 7 140-159 139 139 139 247 247 247 F F F 8 160-179 137 137 137 244 244 244 F F F 9 180-199 137 137 137 244 244 244 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 137 137 137 244 244 244 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 134 134 134 239 239 239 F F F 15 300-319 134 134 134 239 239 239 F F F 16 320-339 137 137 137 244 244 244 F F F 17 340-359 137 137 137 244 244 244 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 137 137 137 244 244 244 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 136 136 136 243 243 243 F F F 22 440-459 134 134 134 239 239 239 F F F 23 460-479 130 130 130 233 233 233 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 138 138 138 246 246 246 F F F 26 520-539 138 138 138 246 246 246 F F F 27 540-559 138 138 138 246 246 246 F F F 28 560-579 130 130 130 233 233 233 F F F 29 580-599 139 139 139 247 247 247 F F F Max gen: 139, Max flush: 247 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.138265586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:31.182747534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.18320829Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.214805144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.215491928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.215526778Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.215546047Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.215560677Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752} {"msg":"Crucible a41384fa-0cd8-45e0-9aab-e6e5b8e424c5 has session id: 6f2ff2cb-cf7a-4345-914d-8908f840253d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216119012Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Upstairs opts: Upstairs UUID: a41384fa-0cd8-45e0-9aab-e6e5b8e424c5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216168962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Crucible stats registered with UUID: a41384fa-0cd8-45e0-9aab-e6e5b8e424c5","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216200501Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:31.216222181Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216605238Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216629777Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752} {"msg":"a41384fa-0cd8-45e0-9aab-e6e5b8e424c5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216678177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216705827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"0","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216736416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"0","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216751676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"1","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216766076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"1","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216778816Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"2","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216802276Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"2","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216833226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216861326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216888205Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45132)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216916445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"ds_connection connected from Ok(127.0.0.1:46015)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216943695Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54553)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.216971194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.217637728Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"2","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.217667488Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"1","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.217923916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"0","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.218438291Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"1","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.218469671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"2","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.21855095Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"0","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219115625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219141435Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[0]R flush_numbers[0..12]: [242, 231, 237, 246, 246, 246, 247, 247, 244, 244, 244, 244]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219207994Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[0]R generation[0..12]: [135, 128, 133, 138, 138, 138, 139, 139, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219228714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219243144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[1]R flush_numbers[0..12]: [242, 231, 237, 246, 246, 246, 247, 247, 244, 244, 244, 244]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219256754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[1]R generation[0..12]: [135, 128, 133, 138, 138, 138, 139, 139, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219270054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219283564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[2]R flush_numbers[0..12]: [242, 231, 237, 246, 246, 246, 247, 247, 244, 244, 244, 244]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219297083Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[2]R generation[0..12]: [135, 128, 133, 138, 138, 138, 139, 139, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219310543Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219324023Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219336633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219349623Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219362173Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219375433Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"":"downstairs","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219388152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219399823Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"0","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219415612Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"1","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219428782Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"client":"2","":"downstairs","session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"All required reconciliation work is completed","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-03-18T01:47:31.219450742Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219462822Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"a41384fa-0cd8-45e0-9aab-e6e5b8e424c5 is now active with session: 6f2ff2cb-cf7a-4345-914d-8908f840253d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219475252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219486992Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752,"session_id":"6f2ff2cb-cf7a-4345-914d-8908f840253d"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219498931Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752} {"msg":"Activated sub_volume a41384fa-0cd8-45e0-9aab-e6e5b8e424c5","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:31.219509882Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 359 len 24576 data: 11 11 9 8 8 7 03/30 Write block 226 len 16384 data: 7 8 9 10 04/30 Flush 05/30 Write block 7 len 8192 data: 14 13 06/30 Read block 211 len 16384 07/30 Read block 486 len 16384 08/30 Write block 522 len 4096 data: 12 09/30 Read block 137 len 12288 10/30 Write block 157 len 4096 data: 6 11/30 Write block 468 len 36864 data: 13 10 10 11 7 6 6 7 8 12/30 Write block 568 len 40960 data: 11 12 9 11 10 8 10 11 10 14 13/30 Read block 405 len 36864 14/30 Write block 455 len 8192 data: 9 9 15/30 Read block 515 len 40960 16/30 Read block 82 len 20480 17/30 Write block 128 len 36864 data: 11 12 9 10 12 13 14 13 12 18/30 Write block 434 len 40960 data: 9 11 10 10 8 10 10 9 10 9 19/30 Read block 318 len 36864 20/30 Flush 21/30 Write block 350 len 12288 data: 14 15 14 22/30 Write block 511 len 24576 data: 12 12 15 15 13 14 23/30 Read block 244 len 24576 24/30 Write block 449 len 12288 data: 11 8 9 25/30 Read block 235 len 20480 26/30 Read block 389 len 16384 27/30 Write block 293 len 8192 data: 13 14 28/30 Read block 333 len 12288 29/30 Write block 220 len 20480 data: 17 19 18 16 13 30/30 Write block 221 len 16384 data: 20 19 17 14 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 3 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:31.239275333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1752} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 138 138 138 246 246 246 F F F 5 100-119 138 138 138 246 246 246 F F F 6 120-139 140 140 140 250 250 250 F F F 7 140-159 140 140 140 250 250 250 F F F 8 160-179 137 137 137 244 244 244 F F F 9 180-199 137 137 137 244 244 244 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 140 140 140 249 249 249 T T T 12 240-259 137 137 137 244 244 244 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 134 134 134 239 239 239 T T T 15 300-319 134 134 134 239 239 239 F F F 16 320-339 137 137 137 244 244 244 F F F 17 340-359 140 140 140 249 249 249 T T T 18 360-379 140 140 140 249 249 249 F F F 19 380-399 137 137 137 244 244 244 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 140 140 140 250 250 250 F F F 22 440-459 140 140 140 250 250 250 T T T 23 460-479 140 140 140 250 250 250 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 138 138 138 246 246 246 T T T 26 520-539 140 140 140 250 250 250 F F F 27 540-559 138 138 138 246 246 246 F F F 28 560-579 140 140 140 250 250 250 F F F 29 580-599 139 139 139 247 247 247 F F F Max gen: 140, Max flush: 250 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 133 133 133 237 237 237 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 138 138 138 246 246 246 F F F 5 100-119 138 138 138 246 246 246 F F F 6 120-139 140 140 140 250 250 250 F F F 7 140-159 140 140 140 250 250 250 F F F 8 160-179 137 137 137 244 244 244 F F F 9 180-199 137 137 137 244 244 244 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 141 141 141 251 251 251 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 141 141 141 251 251 251 F F F 15 300-319 134 134 134 239 239 239 F F F 16 320-339 137 137 137 244 244 244 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 140 140 140 249 249 249 F F F 19 380-399 137 137 137 244 244 244 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 140 140 140 250 250 250 F F F 22 440-459 141 141 141 251 251 251 F F F 23 460-479 140 140 140 250 250 250 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 141 141 141 251 251 251 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 138 138 138 246 246 246 F F F 28 560-579 140 140 140 250 250 250 F F F 29 580-599 139 139 139 247 247 247 F F F Max gen: 141, Max flush: 251 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.514708229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.560538044Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.561375386Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.561428716Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.561443176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.561457816Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760} {"msg":"Crucible d562fc86-8d3d-4d05-bda7-a4ce3431c1f3 has session id: 55f5b20c-917d-4dd4-bccb-390a0d89b672","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.5621395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Upstairs opts: Upstairs UUID: d562fc86-8d3d-4d05-bda7-a4ce3431c1f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562178259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Crucible stats registered with UUID: d562fc86-8d3d-4d05-bda7-a4ce3431c1f3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562203609Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:35.562224739Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562662365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562688135Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760} {"msg":"d562fc86-8d3d-4d05-bda7-a4ce3431c1f3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562760484Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562778694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"0","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562808273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"0","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562831603Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"1","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562854753Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"1","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562879183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"2","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562918352Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"2","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562946112Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.562971402Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.563044042Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"ds_connection connected from Ok(127.0.0.1:56415)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.563085041Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"ds_connection connected from Ok(127.0.0.1:43470)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.563305049Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"ds_connection connected from Ok(127.0.0.1:60793)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.563346219Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.563854864Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"0","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.563897944Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"1","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.564420949Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"2","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.564456199Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"1","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.564619887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"0","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.565128823Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.565244862Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.565264461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"2","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[0]R flush_numbers[0..12]: [250, 231, 237, 246, 246, 246, 250, 250, 244, 244, 244, 251]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.565973545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[0]R generation[0..12]: [140, 128, 133, 138, 138, 138, 140, 140, 137, 137, 137, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566014224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566040724Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[1]R flush_numbers[0..12]: [250, 231, 237, 246, 246, 246, 250, 250, 244, 244, 244, 251]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566067684Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[1]R generation[0..12]: [140, 128, 133, 138, 138, 138, 140, 140, 137, 137, 137, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566094894Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566119884Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[2]R flush_numbers[0..12]: [250, 231, 237, 246, 246, 246, 250, 250, 244, 244, 244, 251]","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2025-03-18T01:47:35.566151693Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[2]R generation[0..12]: [140, 128, 133, 138, 138, 138, 140, 140, 137, 137, 137, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566184273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566206183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566219323Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566232803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566255292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566278962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"":"downstairs","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566303782Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566318502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"0","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566331692Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"1","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566351701Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"client":"2","":"downstairs","session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566375931Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566398261Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"d562fc86-8d3d-4d05-bda7-a4ce3431c1f3 is now active with session: 55f5b20c-917d-4dd4-bccb-390a0d89b672","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566421051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566433671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760,"session_id":"55f5b20c-917d-4dd4-bccb-390a0d89b672"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:35.566449761Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760} {"msg":"Activated sub_volume d562fc86-8d3d-4d05-bda7-a4ce3431c1f3","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T01:47:35.56646952Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 48 len 28672 data: 9 8 9 11 14 11 13 02/30 Read block 330 len 16384 03/30 Read block 23 len 8192 04/30 Write block 592 len 20480 data: 13 16 15 14 12 05/30 Read block 207 len 16384 06/30 Read block 414 len 32768 07/30 Read block 180 len 32768 08/30 Read block 121 len 40960 09/30 Write block 440 len 32768 data: 11 10 11 10 13 12 12 14 10/30 Read block 285 len 28672 11/30 Read block 99 len 4096 12/30 Read block 174 len 32768 13/30 Read block 94 len 28672 14/30 Write block 567 len 28672 data: 12 12 13 10 12 11 9 15/30 Write block 319 len 8192 data: 12 12 16/30 Read block 209 len 40960 17/30 Write block 222 len 24576 data: 20 18 15 9 8 9 18/30 Read block 370 len 20480 19/30 Write block 176 len 16384 data: 8 7 8 9 20/30 Write block 231 len 4096 data: 8 21/30 Flush 22/30 Write block 462 len 8192 data: 10 9 23/30 Read block 448 len 40960 24/30 Read block 322 len 24576 25/30 Write block 186 len 36864 data: 7 8 6 8 7 7 8 7 8 26/30 Read block 147 len 28672 27/30 Write block 40 len 16384 data: 10 11 11 14 28/30 Read block 141 len 40960 29/30 Read block 360 len 20480 30/30 Read block 450 len 40960 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 9 8 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:35.589062046Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1760} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 142 142 142 252 252 252 T T T 3 060-079 138 138 138 246 246 246 F F F 4 080-099 138 138 138 246 246 246 F F F 5 100-119 138 138 138 246 246 246 F F F 6 120-139 140 140 140 250 250 250 F F F 7 140-159 140 140 140 250 250 250 F F F 8 160-179 142 142 142 252 252 252 F F F 9 180-199 137 137 137 244 244 244 T T T 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 141 141 141 251 251 251 F F F 15 300-319 142 142 142 252 252 252 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 140 140 140 249 249 249 F F F 19 380-399 137 137 137 244 244 244 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 140 140 140 250 250 250 F F F 22 440-459 142 142 142 252 252 252 F F F 23 460-479 140 140 140 250 250 250 T T T 24 480-499 137 137 137 244 244 244 F F F 25 500-519 141 141 141 251 251 251 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 138 138 138 246 246 246 F F F 28 560-579 142 142 142 252 252 252 F F F 29 580-599 142 142 142 252 252 252 F F F Max gen: 142, Max flush: 252 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 143 143 143 253 253 253 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 138 138 138 246 246 246 F F F 5 100-119 138 138 138 246 246 246 F F F 6 120-139 140 140 140 250 250 250 F F F 7 140-159 140 140 140 250 250 250 F F F 8 160-179 142 142 142 252 252 252 F F F 9 180-199 143 143 143 253 253 253 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 134 134 134 239 239 239 F F F 14 280-299 141 141 141 251 251 251 F F F 15 300-319 142 142 142 252 252 252 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 140 140 140 249 249 249 F F F 19 380-399 137 137 137 244 244 244 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 140 140 140 250 250 250 F F F 22 440-459 142 142 142 252 252 252 F F F 23 460-479 143 143 143 253 253 253 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 141 141 141 251 251 251 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 138 138 138 246 246 246 F F F 28 560-579 142 142 142 252 252 252 F F F 29 580-599 142 142 142 252 252 252 F F F Max gen: 143, Max flush: 253 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.826391446Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:39.869452357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.869958912Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.901555936Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.90224622Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.90228363Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.902302229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.902316809Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768} {"msg":"Crucible f994f999-bd35-4eb1-b607-bf5cfb461dce has session id: 59147aa6-f795-436b-af39-38d1161fb3ee","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.902918604Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Upstairs opts: Upstairs UUID: f994f999-bd35-4eb1-b607-bf5cfb461dce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.902955034Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Crucible stats registered with UUID: f994f999-bd35-4eb1-b607-bf5cfb461dce","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.902981383Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:39.903003253Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903490379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903523768Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768} {"msg":"f994f999-bd35-4eb1-b607-bf5cfb461dce active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903577668Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903605188Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"0","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903644227Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"0","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903670557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"1","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903699137Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"1","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903724256Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"2","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903751186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"2","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903784346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903814176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903845346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:38408)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903873045Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:52101)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.903901245Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:46591)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.904030184Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.904546859Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"1","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.904618379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"2","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.904757517Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"0","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.905336402Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"1","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.905375482Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"2","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.905403431Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"0","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.905977616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906013926Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[0]R flush_numbers[0..12]: [250, 231, 253, 246, 246, 246, 250, 250, 252, 253, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906068885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[0]R generation[0..12]: [140, 128, 143, 138, 138, 138, 140, 140, 142, 143, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906100695Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906116195Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[1]R flush_numbers[0..12]: [250, 231, 253, 246, 246, 246, 250, 250, 252, 253, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906130505Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[1]R generation[0..12]: [140, 128, 143, 138, 138, 138, 140, 140, 142, 143, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906144775Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906158785Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[2]R flush_numbers[0..12]: [250, 231, 253, 246, 246, 246, 250, 250, 252, 253, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906172194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[2]R generation[0..12]: [140, 128, 143, 138, 138, 138, 140, 140, 142, 143, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906185804Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906199324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906211834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906224794Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906237524Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906250044Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"":"downstairs","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906262784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906274573Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"0","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906289613Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"1","":"downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906303743Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"client":"2","":Wait for a query_work_queue command to finish before sending IO "downstairs","session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906326093Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906338223Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"f994f999-bd35-4eb1-b607-bf5cfb461dce is now active with session: 59147aa6-f795-436b-af39-38d1161fb3ee","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906350843Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906362383Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768,"session_id":"59147aa6-f795-436b-af39-38d1161fb3ee"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906374363Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768} {"msg":"Activated sub_volume f994f999-bd35-4eb1-b607-bf5cfb461dce","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:39.906385332Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 115 len 8192 data: 6 7 02/30 Read block 488 len 12288 03/30 Read block 421 len 4096 04/30 Write block 321 len 20480 data: 12 10 10 11 13 05/30 Read block 236 len 40960 06/30 Write block 106 len 12288 data: 15 14 14 07/30 Write block 264 len 12288 data: 13 13 10 08/30 Write block 323 len 4096 data: 11 09/30 Write block 101 len 28672 data: 13 11 13 9 12 16 15 10/30 Read block 311 len 36864 11/30 Read block 111 len 24576 12/30 Read block 105 len 12288 13/30 Write block 503 len 12288 data: 13 12 11 14/30 Write block 263 len 40960 data: 9 14 14 11 12 10 10 11 11 10 15/30 Flush 16/30 Write block 82 len 36864 data: 13 12 9 8 11 8 11 14 13 17/30 Write block 515 len 8192 data: 14 15 18/30 Read block 191 len 32768 19/30 Write block 188 len 32768 data: 7 9 8 8 9 8 9 5 20/30 Read block 249 len 8192 21/30 Read block 317 len 28672 22/30 Read block 440 len 8192 23/30 Write block 399 len 4096 data: 16 24/30 Flush 25/30 Write block 103 len 40960 data: 14 10 13 17 16 15 13 14 13 11 26/30 Read block 408 len 32768 27/30 Read block 322 len 28672 28/30 Read block 277 len 16384 29/30 Read block 388 len 8192 30/30 Write block 457 len 16384 data: 8 10 8 8 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:39.925955326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1768} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 143 143 143 253 253 253 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 144 144 144 255 255 255 F F F 5 100-119 144 144 144 254 254 254 T T T 6 120-139 140 140 140 250 250 250 F F F 7 140-159 140 140 140 250 250 250 F F F 8 160-179 142 142 142 252 252 252 F F F 9 180-199 144 144 144 255 255 255 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 141 141 141 251 251 251 F F F 15 300-319 142 142 142 252 252 252 F F F 16 320-339 144 144 144 254 254 254 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 140 140 140 249 249 249 F F F 19 380-399 144 144 144 255 255 255 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 140 140 140 250 250 250 F F F 22 440-459 142 142 142 252 252 252 T T T 23 460-479 143 143 143 253 253 253 T T T 24 480-499 137 137 137 244 244 244 F F F 25 500-519 144 144 144 255 255 255 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 138 138 138 246 246 246 F F F 28 560-579 142 142 142 252 252 252 F F F 29 580-599 142 142 142 252 252 252 F F F Max gen: 144, Max flush: 255 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 128 128 128 231 231 231 F F F 2 040-059 143 143 143 253 253 253 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 144 144 144 255 255 255 F F F 5 100-119 145 145 145 256 256 256 F F F 6 120-139 140 140 140 250 250 250 F F F 7 140-159 140 140 140 250 250 250 F F F 8 160-179 142 142 142 252 252 252 F F F 9 180-199 144 144 144 255 255 255 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 141 141 141 251 251 251 F F F 15 300-319 142 142 142 252 252 252 F F F 16 320-339 144 144 144 254 254 254 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 140 140 140 249 249 249 F F F 19 380-399 144 144 144 255 255 255 F F F 20 400-419 136 136 136 243 243 243 F F F 21 420-439 140 140 140 250 250 250 F F F 22 440-459 145 145 145 256 256 256 F F F 23 460-479 145 145 145 256 256 256 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 144 144 144 255 255 255 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 138 138 138 246 246 246 F F F 28 560-579 142 142 142 252 252 252 F F F 29 580-599 142 142 142 252 252 252 F F F Max gen: 145, Max flush: 256 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.188490698Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.232731998Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.233356612Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.233400272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.233415892Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.233432481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776} {"msg":"Crucible 9fac9666-9b93-4d11-b55a-3ed3319e4a83 has session id: fc954957-9e5c-4fdd-9af3-9feafd4774c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234114665Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Upstairs opts: Upstairs UUID: 9fac9666-9b93-4d11-b55a-3ed3319e4a83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234150925Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Crucible stats registered with UUID: 9fac9666-9b93-4d11-b55a-3ed3319e4a83","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234169315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:44.234189755Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.23469304Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.23471814Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776} {"msg":"9fac9666-9b93-4d11-b55a-3ed3319e4a83 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234793909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234812129Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"0","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234846499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"0","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234861788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"1","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234882768Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"1","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234907908Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"2","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234927198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"2","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234941528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234966338Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.234992507Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:65477)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.235018497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:65396)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.235141196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:56055)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.235267695Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.23575232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"2","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.23578284Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"0","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.236360145Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"0","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.236528084Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"2","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.236986119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.237088558Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.237231827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"1","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.237731162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"1","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[0]R flush_numbers[0..12]: [250, 231, 253, 246, 255, 256, 250, 250, 252, 255, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238249418Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[0]R generation[0..12]: [140, 128, 143, 138, 144, 145, 140, 140, 142, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238270638Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238285088Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[1]R flush_numbers[0..12]: [250, 231, 253, 246, 255, 256, 250, 250, 252, 255, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238298688Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[1]R generation[0..12]: [140, 128, 143, 138, 144, 145, 140, 140, 142, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238312247Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238325987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[2]R flush_numbers[0..12]: [250, 231, 253, 246, 255, 256, 250, 250, 252, 255, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238339627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[2]R generation[0..12]: [140, 128, 143, 138, 144, 145, 140, 140, 142, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238353537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238367227Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238384277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238397767Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":Wait for a query_work_queue command to finish before sending IO "Next flush: 257","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238418416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238431236Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"":"downstairs","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238444406Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238456216Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"0","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238469416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"1","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238482566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"client":"2","":"downstairs","session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238495596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238507655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"9fac9666-9b93-4d11-b55a-3ed3319e4a83 is now active with session: fc954957-9e5c-4fdd-9af3-9feafd4774c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238520155Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238531995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776,"session_id":"fc954957-9e5c-4fdd-9af3-9feafd4774c2"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238544165Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776} {"msg":"Activated sub_volume 9fac9666-9b93-4d11-b55a-3ed3319e4a83","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:44.238555125Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 22 len 32768 data: 10 7 8 10 13 15 17 14 02/30 Read block 506 len 36864 03/30 Flush 04/30 Write block 415 len 40960 data: 9 11 10 11 12 13 10 9 8 9 05/30 Flush 06/30 Write block 584 len 28672 data: 11 8 7 10 9 8 9 07/30 Write block 412 len 16384 data: 11 10 9 10 08/30 Flush 09/30 Write block 169 len 16384 data: 7 7 6 7 10/30 Write block 145 len 40960 data: 10 10 10 11 10 10 10 10 10 10 11/30 Read block 266 len 4096 12/30 Read block 311 len 20480 13/30 Read block 552 len 12288 14/30 Write block 529 len 12288 data: 13 12 9 15/30 Read block 588 len 36864 16/30 Flush 17/30 Write block 319 len 16384 data: 13 13 13 11 18/30 Write block 406 len 32768 data: 10 7 7 7 8 11 12 11 19/30 Read block 179 len 40960 20/30 Flush 21/30 Write block 559 len 24576 data: 8 7 6 6 8 9 22/30 Write block 533 len 36864 data: 11 11 13 10 9 11 12 11 12 23/30 Write block 583 len 24576 data: 10 12 9 8 11 10 24/30 Write block 557 len 4096 data: 13 25/30 Read block 186 len 36864 26/30 Read block 232 len 36864 27/30 Write block 448 len 28672 data: 12 12 9 10 12 10 8 28/30 Read block 39 len 16384 29/30 Read block 302 len 12288 30/30 Write block 299 len 20480 data: 6 8 10 13 11 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:44.259558625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1776} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 146 146 146 257 257 257 F F F 2 040-059 143 143 143 253 253 253 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 144 144 144 255 255 255 F F F 5 100-119 145 145 145 256 256 256 F F F 6 120-139 140 140 140 250 250 250 F F F 7 140-159 146 146 146 260 260 260 F F F 8 160-179 146 146 146 260 260 260 F F F 9 180-199 144 144 144 255 255 255 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 141 141 141 251 251 251 T T T 15 300-319 146 146 146 261 261 261 T T T 16 320-339 146 146 146 261 261 261 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 140 140 140 249 249 249 F F F 19 380-399 144 144 144 255 255 255 F F F 20 400-419 146 146 146 261 261 261 F F F 21 420-439 146 146 146 258 258 258 F F F 22 440-459 145 145 145 256 256 256 T T T 23 460-479 145 145 145 256 256 256 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 144 144 144 255 255 255 F F F 26 520-539 146 146 146 260 260 260 T T T 27 540-559 138 138 138 246 246 246 T T T 28 560-579 142 142 142 252 252 252 T T T 29 580-599 146 146 146 259 259 259 T T T Max gen: 146, Max flush: 261 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 146 146 146 257 257 257 F F F 2 040-059 143 143 143 253 253 253 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 144 144 144 255 255 255 F F F 5 100-119 145 145 145 256 256 256 F F F 6 120-139 140 140 140 250 250 250 F F F 7 140-159 146 146 146 260 260 260 F F F 8 160-179 146 146 146 260 260 260 F F F 9 180-199 144 144 144 255 255 255 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 137 137 137 244 244 244 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 147 147 147 262 262 262 F F F 15 300-319 147 147 147 262 262 262 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 140 140 140 249 249 249 F F F 19 380-399 144 144 144 255 255 255 F F F 20 400-419 146 146 146 261 261 261 F F F 21 420-439 146 146 146 258 258 258 F F F 22 440-459 147 147 147 262 262 262 F F F 23 460-479 145 145 145 256 256 256 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 144 144 144 255 255 255 F F F 26 520-539 147 147 147 262 262 262 F F F 27 540-559 147 147 147 262 262 262 F F F 28 560-579 147 147 147 262 262 262 F F F 29 580-599 147 147 147 262 262 262 F F F Max gen: 147, Max flush: 262 Begin loop 74 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.518669809Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.562570602Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.563210516Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.563254405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.563269345Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.563284005Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784} {"msg":"Crucible 0ed9ed4a-d011-42ab-8a7c-bb58d00cbd24 has session id: 224f1b82-9c50-4676-9a37-e07aa425b107","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.56386184Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Upstairs opts: Upstairs UUID: 0ed9ed4a-d011-42ab-8a7c-bb58d00cbd24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.56390104Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Crucible stats registered with UUID: 0ed9ed4a-d011-42ab-8a7c-bb58d00cbd24","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.56391839Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:48.563931679Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564457395Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564527994Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784} {"msg":"0ed9ed4a-d011-42ab-8a7c-bb58d00cbd24 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564545944Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564558554Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"0","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564587724Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"0","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564601333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"1","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564617373Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"1","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564641153Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"2","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564667453Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"2","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564694663Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564725102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564755162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"ds_connection connected from Ok(127.0.0.1:48439)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.564808811Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"ds_connection connected from Ok(127.0.0.1:62859)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.565130379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"ds_connection connected from Ok(127.0.0.1:60080)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.565189188Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.565675764Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"1","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.565714873Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"0","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.565986891Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"2","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.566508476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"0","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.566548956Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"1","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.566818853Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"2","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567129361Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567229329Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[0]R flush_numbers[0..12]: [250, 257, 253, 246, 255, 256, 250, 260, 260, 255, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567350508Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[0]R generation[0..12]: [140, 146, 143, 138, 144, 145, 140, 146, 146, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567371228Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567385928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[1]R flush_numbers[0..12]: [250, 257, 253, 246, 255, 256, 250, 260, 260, 255, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567404658Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[1]R generation[0..12]: [140, 146, 143, 138, 144, 145, 140, 146, 146, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567418738Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567432578Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[2]R flush_numbers[0..12]: [250, 257, 253, 246, 255, 256, 250, 260, 260, 255, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567446748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[2]R generation[0..12]: [140, 146, 143, 138, 144, 145, 140, 146, 146, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567460387Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567474257Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567486987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567499847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:47:48.567512447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567529067Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"":"downstairs","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567541957Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567553657Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"0","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567566646Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"1","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567579437Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"client":"2","":"downstairs","session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567594146Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567606456Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"0ed9ed4a-d011-42ab-8a7c-bb58d00cbd24 is now active with session: 224f1b82-9c50-4676-9a37-e07aa425b107","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567619146Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567630706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784,"session_id":"224f1b82-9c50-4676-9a37-e07aa425b107"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567642676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784} {"msg":"Activated sub_volume 0ed9ed4a-d011-42ab-8a7c-bb58d00cbd24","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:48.567653516Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 20480 data: 12 14 15 15 14 02/30 Write block 378 len 28672 data: 16 13 10 8 10 11 11 03/30 Read block 136 len 24576 04/30 Read block 328 len 20480 05/30 Write block 43 len 20480 data: 15 13 10 9 9 06/30 Read block 188 len 24576 07/30 Read block 549 len 8192 08/30 Read block 355 len 4096 09/30 Write block 415 len 16384 data: 11 12 11 12 10/30 Read block 130 len 20480 11/30 Read block 245 len 12288 12/30 Write block 249 len 28672 data: 8 8 13 15 16 16 15 13/30 Read block 166 len 24576 14/30 Write block 28 len 40960 data: 18 15 13 11 12 10 9 8 9 10 15/30 Read block 246 len 32768 16/30 Write block 578 len 36864 data: 16 19 15 11 9 11 13 10 9 17/30 Read block 57 len 4096 18/30 Read block 416 len 4096 19/30 Flush 20/30 Write block 176 len 16384 data: 9 8 9 10 21/30 Write block 426 len 4096 data: 13 22/30 Write block 357 len 16384 data: 11 11 12 12 23/30 Read block 169 len 8192 24/30 Read block 523 len 16384 25/30 Write block 135 len 16384 data: 14 13 14 12 26/30 Read block 287 len 20480 27/30 Write block 516 len 36864 data: 16 16 12 14 12 15 13 11 14 28/30 Write block 128 len 24576 data: 12 13 10 11 13 14 29/30 Write block 288 len 24576 data: 11 12 15 14 14 14 30/30 Write block 181 len 32768 data: 7 8 6 4 6 8 9 8 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:48.588667046Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1784} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 148 148 148 263 263 263 F F F 2 040-059 148 148 148 263 263 263 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 144 144 144 255 255 255 F F F 5 100-119 145 145 145 256 256 256 F F F 6 120-139 140 140 140 250 250 250 T T T 7 140-159 146 146 146 260 260 260 F F F 8 160-179 146 146 146 260 260 260 T T T 9 180-199 144 144 144 255 255 255 T T T 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 148 148 148 263 263 263 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 147 147 147 262 262 262 T T T 15 300-319 147 147 147 262 262 262 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 141 141 141 251 251 251 T T T 18 360-379 148 148 148 263 263 263 T T T 19 380-399 148 148 148 263 263 263 F F F 20 400-419 148 148 148 263 263 263 F F F 21 420-439 146 146 146 258 258 258 T T T 22 440-459 147 147 147 262 262 262 F F F 23 460-479 145 145 145 256 256 256 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 144 144 144 255 255 255 T T T 26 520-539 147 147 147 262 262 262 T T T 27 540-559 147 147 147 262 262 262 F F F 28 560-579 148 148 148 263 263 263 F F F 29 580-599 148 148 148 263 263 263 F F F Max gen: 148, Max flush: 263 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 250 250 250 F F F 1 020-039 148 148 148 263 263 263 F F F 2 040-059 148 148 148 263 263 263 F F F 3 060-079 138 138 138 246 246 246 F F F 4 080-099 144 144 144 255 255 255 F F F 5 100-119 145 145 145 256 256 256 F F F 6 120-139 149 149 149 264 264 264 F F F 7 140-159 146 146 146 260 260 260 F F F 8 160-179 149 149 149 264 264 264 F F F 9 180-199 149 149 149 264 264 264 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 148 148 148 263 263 263 F F F 13 260-279 144 144 144 254 254 254 F F F 14 280-299 149 149 149 264 264 264 F F F 15 300-319 147 147 147 262 262 262 F F F 16 320-339 146 146 146 261 261 261 F F F 17 340-359 149 149 149 264 264 264 F F F 18 360-379 149 149 149 264 264 264 F F F 19 380-399 148 148 148 263 263 263 F F F 20 400-419 148 148 148 263 263 263 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 147 147 147 262 262 262 F F F 23 460-479 145 145 145 256 256 256 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 149 149 149 264 264 264 F F F 26 520-539 149 149 149 264 264 264 F F F 27 540-559 147 147 147 262 262 262 F F F 28 560-579 148 148 148 263 263 263 F F F 29 580-599 148 148 148 263 263 263 F F F Max gen: 149, Max flush: 264 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.85429738Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.899849678Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.900622241Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.9006865Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.9007043Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.90071865Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792} {"msg":"Crucible 629df1d0-c26e-4ed9-896c-ca44a4f10de6 has session id: 81fe6836-a338-4d3b-a948-a07e01365b1b","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.901330734Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Upstairs opts: Upstairs UUID: 629df1d0-c26e-4ed9-896c-ca44a4f10de6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.901369984Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Crucible stats registered with UUID: 629df1d0-c26e-4ed9-896c-ca44a4f10de6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.901392964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:52.901413064Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.90183673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.90186334Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792} {"msg":"629df1d0-c26e-4ed9-896c-ca44a4f10de6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.901929629Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.901949069Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"0","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.901978408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"0","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.901996518Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"1","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902010978Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"1","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902030668Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"2","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902053588Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"2","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902076288Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902099987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902124187Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47622)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902148167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44216)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902172557Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"ds_connection connected from Ok(127.0.0.1:60369)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.902197986Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.90295638Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"0","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.903024989Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"2","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.903149348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"1","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.903668693Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"0","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.903712823Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"1","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.903779922Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"2","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904397077Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904512586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[0]R flush_numbers[0..12]: [250, 263, 263, 246, 255, 256, 264, 260, 264, 264, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904589905Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[0]R generation[0..12]: [140, 148, 148, 138, 144, 145, 149, 146, 149, 149, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904610615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904625045Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[1]R flush_numbers[0..12]: [250, 263, 263, 246, 255, 256, 264, 260, 264, 264, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904638564Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[1]R generation[0..12]: [140, 148, 148, 138, 144, 145, 149, 146, 149, 149, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904652254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904666054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[2]R flush_numbers[0..12]: [250, 263, 263, 246, 255, 256, 264, 260, 264, 264, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904679404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[2]R generation[0..12]: [140, 148, 148, 138, 144, 145, 149, 146, 149, 149, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904692844Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904706354Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904718974Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904734814Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904747983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904760603Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"":"downstairs","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904782073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904793983Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"0","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904807033Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"1","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904819893Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"client":"2","":"downstairs","session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904832943Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904844793Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"629df1d0-c26e-4ed9-896c-ca44a4f10de6 is now active with session: 81fe6836-a338-4d3b-a948-a07e01365b1b","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904857233Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904868913Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792,"session_id":"81fe6836-a338-4d3b-a948-a07e01365b1b"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904880832Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792} {"msg":"Activated sub_volume 629df1d0-c26e-4ed9-896c-ca44a4f10de6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:52.904891692Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 574 len 40960 data: 11 12 11 15 17 20 16 12 10 12 02/30 Write block 9 len 20480 data: 11 11 13 12 13 03/30 Read block 277 len 8192 04/30 Flush 05/30 Write block 150 len 32768 data: 11 11 11 11 11 11 10 7 06/30 Write block 467 len 24576 data: 12 14 11 11 12 8 07/30 Read block 351 len 28672 08/30 Read block 399 len 4096 09/30 Write block 501 len 12288 data: 13 12 14 10/30 Flush 11/30 Write block 320 len 16384 data: 14 14 12 12 12/30 Read block 212 len 24576 13/30 Flush 14/30 Write block 59 len 28672 data: 6 11 15 15 13 13 16 15/30 Read block 589 len 4096 16/30 Write block 493 len 20480 data: 14 14 12 8 11 17/30 Read block 121 len 4096 18/30 Read block 305 len 28672 19/30 Read block 126 len 16384 20/30 Write block 15 len 24576 data: 13 14 15 14 15 14 21/30 Flush 22/30 Write block 443 len 32768 data: 11 14 13 13 15 13 13 10 23/30 Read block 168 len 40960 24/30 Read block 327 len 16384 25/30 Write block 97 len 16384 data: 9 11 11 13 26/30 Read block 241 len 20480 27/30 Write block 2 len 16384 data: 5 7 8 9 28/30 Write block 276 len 32768 data: 12 14 17 16 18 18 19 16 29/30 Read block 38 len 12288 30/30 Write block 118 len 40960 data: 11 12 17 16 16 15 14 12 12 13 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 8 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:52.926219419Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1792} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 268 268 268 T T T 1 020-039 150 150 150 268 268 268 F F F 2 040-059 150 150 150 268 268 268 F F F 3 060-079 150 150 150 268 268 268 F F F 4 080-099 144 144 144 255 255 255 T T T 5 100-119 145 145 145 256 256 256 T T T 6 120-139 149 149 149 264 264 264 T T T 7 140-159 150 150 150 266 266 266 F F F 8 160-179 149 149 149 264 264 264 F F F 9 180-199 149 149 149 264 264 264 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 148 148 148 263 263 263 F F F 13 260-279 144 144 144 254 254 254 T T T 14 280-299 149 149 149 264 264 264 T T T 15 300-319 147 147 147 262 262 262 F F F 16 320-339 150 150 150 267 267 267 F F F 17 340-359 149 149 149 264 264 264 F F F 18 360-379 149 149 149 264 264 264 F F F 19 380-399 148 148 148 263 263 263 F F F 20 400-419 148 148 148 263 263 263 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 147 147 147 262 262 262 T T T 23 460-479 150 150 150 266 266 266 F F F 24 480-499 150 150 150 268 268 268 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 149 149 149 264 264 264 F F F 27 540-559 147 147 147 262 262 262 F F F 28 560-579 150 150 150 265 265 265 F F F 29 580-599 150 150 150 265 265 265 F F F Max gen: 150, Max flush: 268 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 269 269 269 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 150 150 150 268 268 268 F F F 3 060-079 150 150 150 268 268 268 F F F 4 080-099 151 151 151 269 269 269 F F F 5 100-119 151 151 151 269 269 269 F F F 6 120-139 151 151 151 269 269 269 F F F 7 140-159 150 150 150 266 266 266 F F F 8 160-179 149 149 149 264 264 264 F F F 9 180-199 149 149 149 264 264 264 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 148 148 148 263 263 263 F F F 13 260-279 151 151 151 269 269 269 F F F 14 280-299 151 151 151 269 269 269 F F F 15 300-319 147 147 147 262 262 262 F F F 16 320-339 150 150 150 267 267 267 F F F 17 340-359 149 149 149 264 264 264 F F F 18 360-379 149 149 149 264 264 264 F F F 19 380-399 148 148 148 263 263 263 F F F 20 400-419 148 148 148 263 263 263 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 150 150 150 266 266 266 F F F 24 480-499 150 150 150 268 268 268 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 149 149 149 264 264 264 F F F 27 540-559 147 147 147 262 262 262 F F F 28 560-579 150 150 150 265 265 265 F F F 29 580-599 150 150 150 265 265 265 F F F Max gen: 151, Max flush: 269 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.180884863Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:57.224422379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.224777776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.256554368Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.257263322Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.257305372Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.257323621Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.257341241Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800} {"msg":"Crucible 5828ecdb-7f3f-4716-8c36-173c4836a8d0 has session id: be09e215-8935-474f-bfdf-fd229fc3cdb0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258008485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Upstairs opts: Upstairs UUID: 5828ecdb-7f3f-4716-8c36-173c4836a8d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258049565Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Crucible stats registered with UUID: 5828ecdb-7f3f-4716-8c36-173c4836a8d0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258074654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:57.258090664Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258506301Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.25853101Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800} {"msg":"5828ecdb-7f3f-4716-8c36-173c4836a8d0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.25858915Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.25860706Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"0","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258636469Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"0","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258650499Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"1","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258667979Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"1","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258691639Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"2","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258716569Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"2","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258741799Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258764568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258783328Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59226)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258832698Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46441)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258864987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"ds_connection connected from Ok(127.0.0.1:38673)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.258890497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.259605601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"2","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.259809149Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"1","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.259850158Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"0","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.260317494Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"2","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.260413863Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"1","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.260611002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"0","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.260977638Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261069887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[0]R flush_numbers[0..12]: [269, 268, 268, 268, 269, 269, 269, 266, 264, 264, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261188316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[0]R generation[0..12]: [151, 150, 150, 150, 151, 151, 151, 150, 149, 149, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261208226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261226076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[1]R flush_numbers[0..12]: [269, 268, 268, 268, 269, 269, 269, 266, 264, 264, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261240266Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[1]R generation[0..12]: [151, 150, 150, 150, 151, 151, 151, 150, 149, 149, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261253896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261267586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[2]R flush_numbers[0..12]: [269, 268, 268, 268, 269, 269, 269, 266, 264, 264, 244, 252]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261281025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[2]R generation[0..12]: [151, 150, 150, 150, 151, 151, 151, 150, 149, 149, 137, 142]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261294725Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261308295Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261320935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261333815Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261346305Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:47:57.261358855Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"":"downstairs","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261377455Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261389164Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"0","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261402064Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"1","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261414854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"client":"2","":"downstairs","session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261429714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261441964Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"5828ecdb-7f3f-4716-8c36-173c4836a8d0 is now active with session: be09e215-8935-474f-bfdf-fd229fc3cdb0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261454474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261465984Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800,"session_id":"be09e215-8935-474f-bfdf-fd229fc3cdb0"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261477874Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800} {"msg":"Activated sub_volume 5828ecdb-7f3f-4716-8c36-173c4836a8d0","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:47:57.261488914Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 235 len 4096 data: 13 02/30 Write block 248 len 8192 data: 8 9 03/30 Read block 40 len 20480 04/30 Read block 363 len 12288 05/30 Read block 478 len 36864 06/30 Flush 07/30 Write block 392 len 16384 data: 18 16 14 13 08/30 Read block 203 len 4096 09/30 Write block 471 len 12288 data: 13 9 7 10/30 Flush 11/30 Write block 327 len 16384 data: 12 13 14 15 12/30 Read block 459 len 32768 13/30 Write block 120 len 36864 data: 18 17 17 16 15 13 13 14 13 14/30 Read block 318 len 4096 15/30 Write block 129 len 16384 data: 14 11 12 14 16/30 Write block 393 len 36864 data: 17 15 14 19 22 17 17 16 16 17/30 Write block 561 len 28672 data: 7 7 9 10 12 10 13 18/30 Flush 19/30 Write block 349 len 4096 data: 9 20/30 Read block 266 len 4096 21/30 Read block 115 len 40960 22/30 Write block 64 len 36864 data: 14 17 19 16 17 17 13 11 11 23/30 Write block 534 len 36864 data: 12 14 11 10 12 13 12 13 11 24/30 Read block 79 len 20480 25/30 Write block 98 len 16384 data: 12 12 14 14 26/30 Read block 138 len 32768 27/30 Read block 197 len 12288 28/30 Write block 79 len 36864 data: 12 13 14 14 13 10 9 12 9 29/30 Write block 189 len 8192 data: 10 9 30/30 Read block 330 len 8192 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 1 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 9 Sent Done Sent false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 3 6 Done 9 12 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:47:57.281293024Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1800} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 269 269 269 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 150 150 150 268 268 268 F F F 3 060-079 150 150 150 268 268 268 T T T 4 080-099 151 151 151 269 269 269 T T T 5 100-119 151 151 151 269 269 269 T T T 6 120-139 152 152 152 272 272 272 F F F 7 140-159 150 150 150 266 266 266 F F F 8 160-179 149 149 149 264 264 264 F F F 9 180-199 149 149 149 264 264 264 T T T 10 200-219 137 137 137 244 244 244 F F F 11 220-239 152 152 152 270 270 270 F F F 12 240-259 152 152 152 270 270 270 F F F 13 260-279 151 151 151 269 269 269 F F F 14 280-299 151 151 151 269 269 269 F F F 15 300-319 147 147 147 262 262 262 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 149 149 149 264 264 264 T T T 18 360-379 149 149 149 264 264 264 F F F 19 380-399 152 152 152 272 272 272 F F F 20 400-419 152 152 152 272 272 272 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 152 152 152 271 271 271 F F F 24 480-499 150 150 150 268 268 268 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 149 149 149 264 264 264 T T T 27 540-559 147 147 147 262 262 262 T T T 28 560-579 152 152 152 272 272 272 F F F 29 580-599 150 150 150 265 265 265 F F F Max gen: 152, Max flush: 272 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 269 269 269 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 150 150 150 268 268 268 F F F 3 060-079 153 153 153 273 273 273 F F F 4 080-099 153 153 153 273 273 273 F F F 5 100-119 153 153 153 273 273 273 F F F 6 120-139 152 152 152 272 272 272 F F F 7 140-159 150 150 150 266 266 266 F F F 8 160-179 149 149 149 264 264 264 F F F 9 180-199 153 153 153 273 273 273 F F F 10 200-219 137 137 137 244 244 244 F F F 11 220-239 152 152 152 270 270 270 F F F 12 240-259 152 152 152 270 270 270 F F F 13 260-279 151 151 151 269 269 269 F F F 14 280-299 151 151 151 269 269 269 F F F 15 300-319 147 147 147 262 262 262 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 153 153 153 273 273 273 F F F 18 360-379 149 149 149 264 264 264 F F F 19 380-399 152 152 152 272 272 272 F F F 20 400-419 152 152 152 272 272 272 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 151 151 151 269 269 269 F F F 23 460-479 152 152 152 271 271 271 F F F 24 480-499 150 150 150 268 268 268 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 153 153 153 273 273 273 F F F 27 540-559 153 153 153 273 273 273 F F F 28 560-579 152 152 152 272 272 272 F F F 29 580-599 150 150 150 265 265 265 F F F Max gen: 153, Max flush: 273 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.555571311Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.599667562Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.600248807Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.600293057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.600308686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.600323356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808} {"msg":"Crucible 565a8692-ab85-4cb1-8172-347638d0fe5f has session id: 952a511f-1d2f-4e9d-b1be-a5af8d75554c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.600841192Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Upstairs opts: Upstairs UUID: 565a8692-ab85-4cb1-8172-347638d0fe5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.600881451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Crucible stats registered with UUID: 565a8692-ab85-4cb1-8172-347638d0fe5f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.600906731Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:01.600929051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601361497Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601397076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808} {"msg":"565a8692-ab85-4cb1-8172-347638d0fe5f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601424696Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601448126Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"0","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601510235Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"0","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601546795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"1","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601583365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"1","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601616474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"2","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601650404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"2","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601682414Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601714274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601745943Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53550)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601820343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48493)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601861862Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58530)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.601936282Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.602455017Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"0","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.602493606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"1","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.603107491Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"0","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.60326423Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"1","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.603735745Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.603761875Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"2","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.603792055Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.60433927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"2","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[0]R flush_numbers[0..12]: [269, 268, 268, 273, 273, 273, 272, 266, 264, 273, 244, 270]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.604891105Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[0]R generation[0..12]: [151, 150, 150, 153, 153, 153, 152, 150, 149, 153, 137, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.604911425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.604926224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[1]R flush_numbers[0..12]: [269, 268, 268, 273, 273, 273, 272, 266, 264, 273, 244, 270]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.604940324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[1]R generation[0..12]: [151, 150, 150, 153, 153, 153, 152, 150, 149, 153, 137, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.604953724Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.604967204Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[2]R flush_numbers[0..12]: [269, 268, 268, 273, 273, 273, 272, 266, 264, 273, 244, 270]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.604980554Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[2]R generation[0..12]: [151, 150, 150, 153, 153, 153, 152, 150, 149, 153, 137, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.604996854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605019224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605037743Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605051283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605064083Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605076643Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"":"downstairs","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605089263Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:48:01.605100843Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"0","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605119693Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"1","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605132513Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"client":"2","":"downstairs","session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605145282Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605157193Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"565a8692-ab85-4cb1-8172-347638d0fe5f is now active with session: 952a511f-1d2f-4e9d-b1be-a5af8d75554c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605169522Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605181012Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808,"session_id":"952a511f-1d2f-4e9d-b1be-a5af8d75554c"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605193022Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808} {"msg":"Activated sub_volume 565a8692-ab85-4cb1-8172-347638d0fe5f","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:01.605203842Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 363 len 32768 data: 9 8 6 5 4 4 7 7 02/30 Read block 33 len 24576 03/30 Write block 174 len 24576 data: 8 10 10 9 10 11 04/30 Write block 551 len 16384 data: 15 14 15 12 05/30 Write block 459 len 40960 data: 9 9 9 11 10 11 12 12 13 15 06/30 Read block 479 len 32768 07/30 Write block 227 len 40960 data: 10 10 11 9 9 7 10 12 14 10 08/30 Read block 34 len 36864 09/30 Flush 10/30 Write block 337 len 20480 data: 19 14 13 14 14 11/30 Write block 113 len 40960 data: 10 9 7 8 10 12 13 19 18 18 12/30 Flush 13/30 Write block 185 len 32768 data: 7 9 10 9 11 10 9 10 14/30 Read block 331 len 40960 15/30 Read block 442 len 24576 16/30 Write block 166 len 4096 data: 4 17/30 Read block 385 len 40960 18/30 Write block 79 len 24576 data: 13 14 15 15 14 11 19/30 Read block 397 len 16384 20/30 Write block 199 len 20480 data: 10 11 8 9 10 21/30 Read block 597 len 8192 22/30 Write block 137 len 36864 data: 15 13 11 11 11 10 9 11 11 23/30 Flush 24/30 Write block 492 len 8192 data: 12 15 25/30 Write block 247 len 8192 data: 7 9 26/30 Read block 132 len 16384 27/30 Write block 260 len 28672 data: 8 6 9 10 15 15 12 28/30 Write block 116 len 24576 data: 9 11 13 14 20 19 29/30 Read block 21 len 16384 30/30 Write block 224 len 24576 data: 16 10 9 11 11 12 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:01.626159772Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1808} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 269 269 269 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 150 150 150 268 268 268 F F F 3 060-079 154 154 154 276 276 276 F F F 4 080-099 154 154 154 276 276 276 F F F 5 100-119 154 154 154 275 275 275 T T T 6 120-139 154 154 154 276 276 276 T T T 7 140-159 154 154 154 276 276 276 F F F 8 160-179 154 154 154 276 276 276 F F F 9 180-199 154 154 154 276 276 276 F F F 10 200-219 154 154 154 276 276 276 F F F 11 220-239 154 154 154 274 274 274 T T T 12 240-259 152 152 152 270 270 270 T T T 13 260-279 151 151 151 269 269 269 T T T 14 280-299 151 151 151 269 269 269 F F F 15 300-319 147 147 147 262 262 262 F F F 16 320-339 154 154 154 275 275 275 F F F 17 340-359 154 154 154 275 275 275 F F F 18 360-379 154 154 154 274 274 274 F F F 19 380-399 152 152 152 272 272 272 F F F 20 400-419 152 152 152 272 272 272 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 154 154 154 274 274 274 F F F 23 460-479 154 154 154 274 274 274 F F F 24 480-499 150 150 150 268 268 268 T T T 25 500-519 150 150 150 266 266 266 F F F 26 520-539 153 153 153 273 273 273 F F F 27 540-559 154 154 154 274 274 274 F F F 28 560-579 152 152 152 272 272 272 F F F 29 580-599 150 150 150 265 265 265 F F F Max gen: 154, Max flush: 276 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 269 269 269 F F F 1 020-039 150 150 150 268 268 268 F F F 2 040-059 150 150 150 268 268 268 F F F 3 060-079 154 154 154 276 276 276 F F F 4 080-099 154 154 154 276 276 276 F F F 5 100-119 155 155 155 277 277 277 F F F 6 120-139 155 155 155 277 277 277 F F F 7 140-159 154 154 154 276 276 276 F F F 8 160-179 154 154 154 276 276 276 F F F 9 180-199 154 154 154 276 276 276 F F F 10 200-219 154 154 154 276 276 276 F F F 11 220-239 155 155 155 277 277 277 F F F 12 240-259 155 155 155 277 277 277 F F F 13 260-279 155 155 155 277 277 277 F F F 14 280-299 151 151 151 269 269 269 F F F 15 300-319 147 147 147 262 262 262 F F F 16 320-339 154 154 154 275 275 275 F F F 17 340-359 154 154 154 275 275 275 F F F 18 360-379 154 154 154 274 274 274 F F F 19 380-399 152 152 152 272 272 272 F F F 20 400-419 152 152 152 272 272 272 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 154 154 154 274 274 274 F F F 23 460-479 154 154 154 274 274 274 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 153 153 153 273 273 273 F F F 27 540-559 154 154 154 274 274 274 F F F 28 560-579 152 152 152 272 272 272 F F F 29 580-599 150 150 150 265 265 265 F F F Max gen: 155, Max flush: 277 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.877992292Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:05.922126622Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.922745076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.95436652Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.955099754Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.955136474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.955149923Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.955164243Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816} {"msg":"Crucible b5fc24d5-9ed3-4203-b173-70dadd3a912c has session id: b102a3f7-534b-4b0c-aff5-3e4414aa0447","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.955776408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Upstairs opts: Upstairs UUID: b5fc24d5-9ed3-4203-b173-70dadd3a912c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.955805797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Crucible stats registered with UUID: b5fc24d5-9ed3-4203-b173-70dadd3a912c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.955820877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:05.955833687Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956263123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956290673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816} {"msg":"b5fc24d5-9ed3-4203-b173-70dadd3a912c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956370002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956389152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"0","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956421442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"0","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956437522Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"1","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956462851Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"1","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956488621Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"2","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956518731Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"2","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956547981Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.95658402Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.95661546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"ds_connection connected from Ok(127.0.0.1:38242)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.95664633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"ds_connection connected from Ok(127.0.0.1:47060)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.95667434Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"ds_connection connected from Ok(127.0.0.1:63811)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.956701429Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.957499092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"1","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.957539762Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"2","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.95770963Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"0","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958200666Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"2","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958239555Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"1","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958376844Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"0","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.9588196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.95884558Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[0]R flush_numbers[0..12]: [269, 268, 268, 276, 276, 277, 277, 276, 276, 276, 276, 277]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958904859Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[0]R generation[0..12]: [151, 150, 150, 154, 154, 155, 155, 154, 154, 154, 154, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958924949Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958943179Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[1]R flush_numbers[0..12]: [269, 268, 268, 276, 276, 277, 277, 276, 276, 276, 276, 277]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958957539Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[1]R generation[0..12]: [151, 150, 150, 154, 154, 155, 155, 154, 154, 154, 154, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958971769Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958985559Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[2]R flush_numbers[0..12]: [269, 268, 268, 276, 276, 277, 277, 276, 276, 276, 276, 277]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.958999068Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[2]R generation[0..12]: [151, 150, 150, 154, 154, 155, 155, 154, 154, 154, 154, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959012498Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959026158Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959038898Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959051788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959064398Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959077018Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"":"downstairs","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959090328Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959101958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"0","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959115158Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"1","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959129987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"client":"2","":"downstairs","session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959143367Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} Wait for a query_work_queue command to finish before sending IO {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959161257Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"b5fc24d5-9ed3-4203-b173-70dadd3a912c is now active with session: b102a3f7-534b-4b0c-aff5-3e4414aa0447","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959173857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959185507Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816,"session_id":"b102a3f7-534b-4b0c-aff5-3e4414aa0447"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959197277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816} {"msg":"Activated sub_volume b5fc24d5-9ed3-4203-b173-70dadd3a912c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:05.959208226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 9 len 40960 data: 12 12 14 13 14 10 14 15 16 15 02/30 Write block 443 len 24576 data: 12 15 14 14 16 14 03/30 Read block 524 len 12288 04/30 Write block 270 len 12288 data: 12 12 11 05/30 Read block 423 len 16384 06/30 Read block 346 len 32768 07/30 Write block 378 len 40960 data: 17 14 11 9 11 12 12 13 12 15 08/30 Write block 378 len 16384 data: 18 15 12 10 09/30 Read block 258 len 16384 10/30 Read block 405 len 8192 11/30 Write block 289 len 32768 data: 13 16 15 15 15 15 12 9 12/30 Flush 13/30 Write block 528 len 36864 data: 11 14 13 10 11 12 13 15 12 14/30 Read block 113 len 24576 15/30 Read block 461 len 40960 16/30 Write block 236 len 4096 data: 11 17/30 Write block 30 len 8192 data: 14 12 18/30 Flush 19/30 Write block 218 len 40960 data: 15 15 18 21 21 19 17 11 10 12 20/30 Read block 53 len 36864 21/30 Read block 271 len 16384 22/30 Write block 471 len 32768 data: 14 10 8 7 8 9 10 12 23/30 Write block 327 len 20480 data: 13 14 15 16 16 24/30 Write block 16 len 32768 data: 16 17 16 16 15 10 11 8 25/30 Read block 278 len 12288 26/30 Write block 8 len 4096 data: 14 27/30 Write block 73 len 36864 data: 12 11 13 12 14 14 14 15 16 28/30 Write block 183 len 12288 data: 7 5 8 29/30 Read block 376 len 24576 30/30 Write block 317 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 6 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 2 6 Done 11 9 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:05.979931729Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1816} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 278 278 278 T T T 1 020-039 156 156 156 279 279 279 T T T 2 040-059 150 150 150 268 268 268 F F F 3 060-079 154 154 154 276 276 276 T T T 4 080-099 154 154 154 276 276 276 T T T 5 100-119 155 155 155 277 277 277 F F F 6 120-139 155 155 155 277 277 277 F F F 7 140-159 154 154 154 276 276 276 F F F 8 160-179 154 154 154 276 276 276 F F F 9 180-199 154 154 154 276 276 276 T T T 10 200-219 154 154 154 276 276 276 T T T 11 220-239 156 156 156 279 279 279 T T T 12 240-259 155 155 155 277 277 277 F F F 13 260-279 156 156 156 278 278 278 F F F 14 280-299 156 156 156 278 278 278 F F F 15 300-319 147 147 147 262 262 262 T T T 16 320-339 154 154 154 275 275 275 T T T 17 340-359 154 154 154 275 275 275 F F F 18 360-379 156 156 156 278 278 278 F F F 19 380-399 156 156 156 278 278 278 F F F 20 400-419 152 152 152 272 272 272 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 156 156 156 278 278 278 F F F 23 460-479 154 154 154 274 274 274 T T T 24 480-499 155 155 155 277 277 277 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 156 156 156 279 279 279 F F F 27 540-559 154 154 154 274 274 274 F F F 28 560-579 152 152 152 272 272 272 F F F 29 580-599 150 150 150 265 265 265 F F F Max gen: 156, Max flush: 279 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 280 280 280 F F F 1 020-039 157 157 157 280 280 280 F F F 2 040-059 150 150 150 268 268 268 F F F 3 060-079 157 157 157 280 280 280 F F F 4 080-099 157 157 157 280 280 280 F F F 5 100-119 155 155 155 277 277 277 F F F 6 120-139 155 155 155 277 277 277 F F F 7 140-159 154 154 154 276 276 276 F F F 8 160-179 154 154 154 276 276 276 F F F 9 180-199 157 157 157 280 280 280 F F F 10 200-219 157 157 157 280 280 280 F F F 11 220-239 157 157 157 280 280 280 F F F 12 240-259 155 155 155 277 277 277 F F F 13 260-279 156 156 156 278 278 278 F F F 14 280-299 156 156 156 278 278 278 F F F 15 300-319 157 157 157 280 280 280 F F F 16 320-339 157 157 157 280 280 280 F F F 17 340-359 154 154 154 275 275 275 F F F 18 360-379 156 156 156 278 278 278 F F F 19 380-399 156 156 156 278 278 278 F F F 20 400-419 152 152 152 272 272 272 F F F 21 420-439 149 149 149 264 264 264 F F F 22 440-459 156 156 156 278 278 278 F F F 23 460-479 157 157 157 280 280 280 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 156 156 156 279 279 279 F F F 27 540-559 154 154 154 274 274 274 F F F 28 560-579 152 152 152 272 272 272 F F F 29 580-599 150 150 150 265 265 265 F F F Max gen: 157, Max flush: 280 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.286052408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.331206629Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.331834073Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.331877673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.331892213Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.331906593Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824} {"msg":"Crucible b8f3f0df-301c-4262-8065-62b1801a3eae has session id: 913b5587-eec0-4bec-83db-e35f5d8c0367","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.332446548Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Upstairs opts: Upstairs UUID: b8f3f0df-301c-4262-8065-62b1801a3eae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.332577827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Crucible stats registered with UUID: b8f3f0df-301c-4262-8065-62b1801a3eae","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.332610196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:10.332625946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333097102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333129542Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824} {"msg":"b8f3f0df-301c-4262-8065-62b1801a3eae active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333214701Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333239021Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"0","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.33326949Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"0","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.33328761Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"1","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.33331222Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"1","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.33333822Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"2","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.33336606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"2","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333392669Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333418969Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333446219Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"ds_connection connected from Ok(127.0.0.1:48575)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333472379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"ds_connection connected from Ok(127.0.0.1:38873)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333496958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"ds_connection connected from Ok(127.0.0.1:48594)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.333729366Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.334149453Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"1","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.334189492Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"0","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.334335711Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"2","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.334828486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"0","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.334856626Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"1","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.334871696Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"2","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335411991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.33551498Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[0]R flush_numbers[0..12]: [280, 280, 268, 280, 280, 277, 277, 276, 276, 280, 280, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.33554889Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[0]R generation[0..12]: [157, 157, 150, 157, 157, 155, 155, 154, 154, 157, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.33556402Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335578409Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[1]R flush_numbers[0..12]: [280, 280, 268, 280, 280, 277, 277, 276, 276, 280, 280, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335592319Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[1]R generation[0..12]: [157, 157, 150, 157, 157, 155, 155, 154, 154, 157, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335606179Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335619849Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[2]R flush_numbers[0..12]: [280, 280, 268, 280, 280, 277, 277, 276, 276, 280, 280, 280]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335633429Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[2]R generation[0..12]: [157, 157, 150, 157, 157, 155, 155, 154, 154, 157, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335647049Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335664969Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335678029Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335691068Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335703588Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335719848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"":"downstairs","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335737818Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335749728Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"0","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335763638Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"1","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335776548Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"client":"2","":"downstairs","session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335789378Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:48:10.335801347Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"b8f3f0df-301c-4262-8065-62b1801a3eae is now active with session: 913b5587-eec0-4bec-83db-e35f5d8c0367","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335820817Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335832427Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824,"session_id":"913b5587-eec0-4bec-83db-e35f5d8c0367"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335844407Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824} {"msg":"Activated sub_volume b8f3f0df-301c-4262-8065-62b1801a3eae","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:10.335855217Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 528 len 20480 data: 12 15 14 11 12 02/30 Write block 452 len 28672 data: 13 11 9 10 10 9 11 03/30 Write block 435 len 16384 data: 12 11 11 9 04/30 Write block 221 len 40960 data: 22 22 20 18 12 11 13 12 13 10 05/30 Read block 367 len 4096 06/30 Read block 209 len 40960 07/30 Write block 389 len 20480 data: 18 20 18 19 18 08/30 Read block 195 len 8192 09/30 Write block 147 len 40960 data: 11 12 11 12 12 12 12 12 12 11 10/30 Read block 40 len 24576 11/30 Write block 401 len 40960 data: 17 17 16 13 10 11 8 8 8 9 12/30 Read block 17 len 20480 13/30 Read block 193 len 16384 14/30 Write block 95 len 32768 data: 11 9 10 13 13 15 15 12 15/30 Write block 556 len 24576 data: 13 14 9 9 8 8 16/30 Write block 151 len 32768 data: 13 13 13 13 13 12 8 6 17/30 Write block 373 len 24576 data: 16 17 17 15 16 19 18/30 Write block 593 len 16384 data: 17 16 15 13 19/30 Flush 20/30 Write block 194 len 4096 data: 10 21/30 Write block 366 len 32768 data: 6 5 5 8 8 10 14 17 22/30 Read block 334 len 24576 23/30 Read block 321 len 20480 24/30 Read block 478 len 32768 25/30 Read block 588 len 4096 26/30 Read block 437 len 40960 27/30 Read block 369 len 20480 28/30 Read block 228 len 40960 29/30 Read block 568 len 36864 30/30 Write block 39 len 28672 data: 9 11 12 12 16 14 11 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:10.356970916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1824} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 280 280 280 F F F 1 020-039 157 157 157 280 280 280 T T T 2 040-059 150 150 150 268 268 268 T T T 3 060-079 157 157 157 280 280 280 F F F 4 080-099 158 158 158 281 281 281 F F F 5 100-119 158 158 158 281 281 281 F F F 6 120-139 155 155 155 277 277 277 F F F 7 140-159 158 158 158 281 281 281 F F F 8 160-179 154 154 154 276 276 276 F F F 9 180-199 157 157 157 280 280 280 T T T 10 200-219 157 157 157 280 280 280 F F F 11 220-239 158 158 158 281 281 281 F F F 12 240-259 155 155 155 277 277 277 F F F 13 260-279 156 156 156 278 278 278 F F F 14 280-299 156 156 156 278 278 278 F F F 15 300-319 157 157 157 280 280 280 F F F 16 320-339 157 157 157 280 280 280 F F F 17 340-359 154 154 154 275 275 275 F F F 18 360-379 158 158 158 281 281 281 T T T 19 380-399 158 158 158 281 281 281 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 158 158 158 281 281 281 F F F 22 440-459 158 158 158 281 281 281 F F F 23 460-479 157 157 157 280 280 280 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 158 158 158 281 281 281 F F F 27 540-559 158 158 158 281 281 281 F F F 28 560-579 158 158 158 281 281 281 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 158, Max flush: 281 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 280 280 280 F F F 1 020-039 159 159 159 282 282 282 F F F 2 040-059 159 159 159 282 282 282 F F F 3 060-079 157 157 157 280 280 280 F F F 4 080-099 158 158 158 281 281 281 F F F 5 100-119 158 158 158 281 281 281 F F F 6 120-139 155 155 155 277 277 277 F F F 7 140-159 158 158 158 281 281 281 F F F 8 160-179 154 154 154 276 276 276 F F F 9 180-199 159 159 159 282 282 282 F F F 10 200-219 157 157 157 280 280 280 F F F 11 220-239 158 158 158 281 281 281 F F F 12 240-259 155 155 155 277 277 277 F F F 13 260-279 156 156 156 278 278 278 F F F 14 280-299 156 156 156 278 278 278 F F F 15 300-319 157 157 157 280 280 280 F F F 16 320-339 157 157 157 280 280 280 F F F 17 340-359 154 154 154 275 275 275 F F F 18 360-379 159 159 159 282 282 282 F F F 19 380-399 158 158 158 281 281 281 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 158 158 158 281 281 281 F F F 22 440-459 158 158 158 281 281 281 F F F 23 460-479 157 157 157 280 280 280 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 150 150 150 266 266 266 F F F 26 520-539 158 158 158 281 281 281 F F F 27 540-559 158 158 158 281 281 281 F F F 28 560-579 158 158 158 281 281 281 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 159, Max flush: 282 Begin loop 80 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.600374732Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.645464324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.646068658Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.646123258Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.646142868Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.646162047Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832} {"msg":"Crucible f30ef63c-df7b-4bb6-a940-24cc2e164dba has session id: bbe0f809-5328-4cc7-a7f8-138b34db6302","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.646625393Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Upstairs opts: Upstairs UUID: f30ef63c-df7b-4bb6-a940-24cc2e164dba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.646665853Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Crucible stats registered with UUID: f30ef63c-df7b-4bb6-a940-24cc2e164dba","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.646710333Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:14.646726162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647205088Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647231418Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832} {"msg":"f30ef63c-df7b-4bb6-a940-24cc2e164dba active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647303527Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647321357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"0","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647350067Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"0","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647371976Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"1","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647398496Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"1","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647425986Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"2","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647448096Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"2","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647462346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647490745Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647517335Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"ds_connection connected from Ok(127.0.0.1:36947)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647543135Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"ds_connection connected from Ok(127.0.0.1:35424)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647638554Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"ds_connection connected from Ok(127.0.0.1:49384)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.647677214Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.648367907Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"1","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.648396037Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"0","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.649100651Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"0","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.64913896Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"1","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.649601916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"2","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.649746255Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.649776425Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650069852Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"2","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[0]R flush_numbers[0..12]: [280, 282, 282, 280, 281, 281, 277, 281, 276, 282, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650593537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[0]R generation[0..12]: [157, 159, 159, 157, 158, 158, 155, 158, 154, 159, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650613787Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650632197Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[1]R flush_numbers[0..12]: [280, 282, 282, 280, 281, 281, 277, 281, 276, 282, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650646237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[1]R generation[0..12]: [157, 159, 159, 157, 158, 158, 155, 158, 154, 159, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650659537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650673047Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[2]R flush_numbers[0..12]: [280, 282, 282, 280, 281, 281, 277, 281, 276, 282, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650705196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[2]R generation[0..12]: [157, 159, 159, 157, 158, 158, 155, 158, 154, 159, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650719376Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650733166Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650745716Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650758556Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650771186Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650783626Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"":"downstairs","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650796765Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650808505Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"0","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650821995Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"1","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650836885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"client":"2","":"downstairs","session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650850025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650862045Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":Wait for a query_work_queue command to finish before sending IO 1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"f30ef63c-df7b-4bb6-a940-24cc2e164dba is now active with session: bbe0f809-5328-4cc7-a7f8-138b34db6302","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650881285Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650893005Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832,"session_id":"bbe0f809-5328-4cc7-a7f8-138b34db6302"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650904784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832} {"msg":"Activated sub_volume f30ef63c-df7b-4bb6-a940-24cc2e164dba","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:14.650915624Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 356 len 12288 data: 9 12 12 02/30 Write block 366 len 40960 data: 7 6 6 9 9 11 15 18 18 18 03/30 Write block 113 len 16384 data: 11 10 8 10 04/30 Flush 05/30 Flush 06/30 Write block 559 len 28672 data: 10 9 9 8 10 11 13 07/30 Read block 49 len 40960 08/30 Read block 198 len 4096 09/30 Read block 497 len 28672 10/30 Write block 377 len 32768 data: 17 20 16 13 11 12 13 13 11/30 Flush 12/30 Write block 128 len 36864 data: 14 15 12 13 15 15 15 15 14 13/30 Flush 14/30 Write block 522 len 4096 data: 14 15/30 Write block 308 len 4096 data: 10 16/30 Write block 516 len 12288 data: 17 17 13 17/30 Write block 297 len 32768 data: 9 8 7 9 11 14 12 10 18/30 Write block 436 len 24576 data: 12 12 10 11 12 11 19/30 Read block 121 len 16384 20/30 Write block 466 len 12288 data: 13 14 16 21/30 Read block 167 len 36864 22/30 Read block 236 len 28672 23/30 Write block 49 len 20480 data: 9 10 12 15 12 24/30 Flush 25/30 Write block 319 len 16384 data: 14 15 15 13 26/30 Write block 249 len 12288 data: 10 9 14 27/30 Read block 193 len 40960 28/30 Read block 227 len 24576 29/30 Write block 255 len 8192 data: 16 13 30/30 Write block 16 len 36864 data: 17 18 17 17 16 11 12 9 9 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:14.670012762Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1832} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 280 280 280 T T T 1 020-039 159 159 159 282 282 282 T T T 2 040-059 160 160 160 287 287 287 F F F 3 060-079 157 157 157 280 280 280 F F F 4 080-099 158 158 158 281 281 281 F F F 5 100-119 160 160 160 283 283 283 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 158 158 158 281 281 281 F F F 8 160-179 154 154 154 276 276 276 F F F 9 180-199 159 159 159 282 282 282 F F F 10 200-219 157 157 157 280 280 280 F F F 11 220-239 158 158 158 281 281 281 F F F 12 240-259 155 155 155 277 277 277 T T T 13 260-279 156 156 156 278 278 278 F F F 14 280-299 160 160 160 287 287 287 F F F 15 300-319 160 160 160 287 287 287 T T T 16 320-339 157 157 157 280 280 280 T T T 17 340-359 160 160 160 283 283 283 F F F 18 360-379 160 160 160 285 285 285 F F F 19 380-399 160 160 160 285 285 285 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 160 160 160 287 287 287 F F F 22 440-459 160 160 160 287 287 287 F F F 23 460-479 160 160 160 287 287 287 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 160 160 160 287 287 287 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 160 160 160 285 285 285 F F F 28 560-579 160 160 160 285 285 285 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 160, Max flush: 287 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 288 288 288 F F F 1 020-039 161 161 161 288 288 288 F F F 2 040-059 160 160 160 287 287 287 F F F 3 060-079 157 157 157 280 280 280 F F F 4 080-099 158 158 158 281 281 281 F F F 5 100-119 160 160 160 283 283 283 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 158 158 158 281 281 281 F F F 8 160-179 154 154 154 276 276 276 F F F 9 180-199 159 159 159 282 282 282 F F F 10 200-219 157 157 157 280 280 280 F F F 11 220-239 158 158 158 281 281 281 F F F 12 240-259 161 161 161 288 288 288 F F F 13 260-279 156 156 156 278 278 278 F F F 14 280-299 160 160 160 287 287 287 F F F 15 300-319 161 161 161 288 288 288 F F F 16 320-339 161 161 161 288 288 288 F F F 17 340-359 160 160 160 283 283 283 F F F 18 360-379 160 160 160 285 285 285 F F F 19 380-399 160 160 160 285 285 285 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 160 160 160 287 287 287 F F F 22 440-459 160 160 160 287 287 287 F F F 23 460-479 160 160 160 287 287 287 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 160 160 160 287 287 287 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 160 160 160 285 285 285 F F F 28 560-579 160 160 160 285 285 285 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 161, Max flush: 288 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.917546439Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.961308643Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.961884298Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.961934358Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.961953678Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.961971417Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840} {"msg":"Crucible b992d12d-5498-457d-b5e6-7ad9e73bd83b has session id: eaec3952-b973-470c-a495-2935806673e1","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.962544252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Upstairs opts: Upstairs UUID: b992d12d-5498-457d-b5e6-7ad9e73bd83b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.962579482Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Crucible stats registered with UUID: b992d12d-5498-457d-b5e6-7ad9e73bd83b","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.962596972Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:18.962612532Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963103587Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963146447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840} {"msg":"b992d12d-5498-457d-b5e6-7ad9e73bd83b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963184237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963201877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"0","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963239496Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"0","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963262736Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"1","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963281976Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"1","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963298416Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"2","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963315836Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"2","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963344085Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963443324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963474114Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:41798)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963489004Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58450)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.963816341Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46092)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.96395226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.964495725Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"0","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.964533714Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"1","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.965125329Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"0","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.965273438Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"1","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.965744994Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.965838183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.966222829Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"2","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.966826154Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"2","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[0]R flush_numbers[0..12]: [288, 288, 287, 280, 281, 283, 286, 281, 276, 282, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967402908Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[0]R generation[0..12]: [161, 161, 160, 157, 158, 160, 160, 158, 154, 159, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967428948Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967444448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[1]R flush_numbers[0..12]: [288, 288, 287, 280, 281, 283, 286, 281, 276, 282, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967458208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[1]R generation[0..12]: [161, 161, 160, 157, 158, 160, 160, 158, 154, 159, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967471648Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967485418Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[2]R flush_numbers[0..12]: [288, 288, 287, 280, 281, 283, 286, 281, 276, 282, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967498647Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[2]R generation[0..12]: [161, 161, 160, 157, 158, 160, 160, 158, 154, 159, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967512098Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967525637Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967538157Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967554897Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967567857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967580227Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"":"downstairs","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967592837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967604537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"0","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967617297Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"1","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967629736Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"client":"2","":"downstairs","session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967642436Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967654356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"b992d12d-5498-457d-b5e6-7ad9e73bd83b is now active with session: eaec3952-b973-470c-a495-2935806673e1","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:48:18.967666756Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967684586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840,"session_id":"eaec3952-b973-470c-a495-2935806673e1"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967696356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840} {"msg":"Activated sub_volume b992d12d-5498-457d-b5e6-7ad9e73bd83b","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:18.967707096Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 546 len 4096 data: 11 02/30 Read block 263 len 12288 03/30 Write block 87 len 12288 data: 10 12 15 04/30 Write block 256 len 12288 data: 14 12 10 05/30 Flush 06/30 Write block 477 len 12288 data: 11 13 7 07/30 Write block 84 len 8192 data: 12 10 08/30 Read block 463 len 4096 09/30 Write block 338 len 32768 data: 15 14 15 15 10 8 8 11 10/30 Write block 88 len 24576 data: 13 16 14 15 14 12 11/30 Write block 82 len 36864 data: 16 15 13 11 13 11 14 17 15 12/30 Flush 13/30 Write block 575 len 12288 data: 13 12 16 14/30 Write block 248 len 24576 data: 10 11 10 15 16 17 15/30 Write block 165 len 4096 data: 3 16/30 Write block 105 len 24576 data: 14 18 17 16 14 15 17/30 Write block 273 len 16384 data: 13 12 12 13 18/30 Write block 424 len 16384 data: 10 10 14 9 19/30 Read block 77 len 40960 20/30 Read block 286 len 28672 21/30 Read block 374 len 24576 22/30 Write block 38 len 16384 data: 9 10 12 13 23/30 Read block 552 len 40960 24/30 Write block 347 len 16384 data: 9 10 10 15 25/30 Write block 350 len 16384 data: 16 16 15 13 26/30 Write block 86 len 24576 data: 14 12 15 18 16 16 27/30 Write block 99 len 36864 data: 14 16 16 13 15 11 15 19 18 28/30 Read block 544 len 20480 29/30 Write block 189 len 12288 data: 12 11 10 30/30 Write block 36 len 24576 data: 10 11 10 11 13 14 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:18.985548324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1840} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 288 288 288 F F F 1 020-039 161 161 161 288 288 288 T T T 2 040-059 160 160 160 287 287 287 T T T 3 060-079 157 157 157 280 280 280 F F F 4 080-099 162 162 162 290 290 290 T T T 5 100-119 160 160 160 283 283 283 T T T 6 120-139 160 160 160 286 286 286 F F F 7 140-159 158 158 158 281 281 281 F F F 8 160-179 154 154 154 276 276 276 T T T 9 180-199 159 159 159 282 282 282 T T T 10 200-219 157 157 157 280 280 280 F F F 11 220-239 158 158 158 281 281 281 F F F 12 240-259 162 162 162 289 289 289 T T T 13 260-279 156 156 156 278 278 278 T T T 14 280-299 160 160 160 287 287 287 F F F 15 300-319 161 161 161 288 288 288 F F F 16 320-339 162 162 162 290 290 290 F F F 17 340-359 162 162 162 290 290 290 T T T 18 360-379 160 160 160 285 285 285 F F F 19 380-399 160 160 160 285 285 285 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 160 160 160 287 287 287 T T T 22 440-459 160 160 160 287 287 287 F F F 23 460-479 162 162 162 290 290 290 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 160 160 160 287 287 287 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 162 162 162 289 289 289 F F F 28 560-579 160 160 160 285 285 285 T T T 29 580-599 158 158 158 281 281 281 F F F Max gen: 162, Max flush: 290 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 288 288 288 F F F 1 020-039 163 163 163 291 291 291 F F F 2 040-059 163 163 163 291 291 291 F F F 3 060-079 157 157 157 280 280 280 F F F 4 080-099 163 163 163 291 291 291 F F F 5 100-119 163 163 163 291 291 291 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 158 158 158 281 281 281 F F F 8 160-179 163 163 163 291 291 291 F F F 9 180-199 163 163 163 291 291 291 F F F 10 200-219 157 157 157 280 280 280 F F F 11 220-239 158 158 158 281 281 281 F F F 12 240-259 163 163 163 291 291 291 F F F 13 260-279 163 163 163 291 291 291 F F F 14 280-299 160 160 160 287 287 287 F F F 15 300-319 161 161 161 288 288 288 F F F 16 320-339 162 162 162 290 290 290 F F F 17 340-359 163 163 163 291 291 291 F F F 18 360-379 160 160 160 285 285 285 F F F 19 380-399 160 160 160 285 285 285 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 163 163 163 291 291 291 F F F 22 440-459 160 160 160 287 287 287 F F F 23 460-479 162 162 162 290 290 290 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 160 160 160 287 287 287 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 162 162 162 289 289 289 F F F 28 560-579 163 163 163 291 291 291 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 163, Max flush: 291 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.256935867Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.303170498Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.303828832Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.303871942Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.303885962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.303900092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848} {"msg":"Crucible 33225d26-ed40-4fee-86c3-db42153693c2 has session id: 51d923c0-7a03-4072-8513-66b62f91de3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.304449827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Upstairs opts: Upstairs UUID: 33225d26-ed40-4fee-86c3-db42153693c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.304492966Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Crucible stats registered with UUID: 33225d26-ed40-4fee-86c3-db42153693c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.304518096Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:23.304539426Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.304995142Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305029992Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848} {"msg":"33225d26-ed40-4fee-86c3-db42153693c2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305133711Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305156071Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"0","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.30518791Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"0","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.30520938Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"1","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.30523673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"1","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305264079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"2","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305293069Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"2","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305319169Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305345149Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305376618Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59932)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305453878Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40018)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305572627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43455)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.305601996Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.30626037Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"0","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.306412999Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"1","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.306487008Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"2","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.306889565Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"0","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.306929415Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"2","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307079563Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"1","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307510479Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307624088Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[0]R flush_numbers[0..12]: [288, 291, 291, 280, 291, 291, 286, 281, 291, 291, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307745817Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[0]R generation[0..12]: [161, 163, 163, 157, 163, 163, 160, 158, 163, 163, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307765117Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307783757Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[1]R flush_numbers[0..12]: [288, 291, 291, 280, 291, 291, 286, 281, 291, 291, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307798317Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[1]R generation[0..12]: [161, 163, 163, 157, 163, 163, 160, 158, 163, 163, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307811976Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307825936Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[2]R flush_numbers[0..12]: [288, 291, 291, 280, 291, 291, 286, 281, 291, 291, 280, 281]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307839606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[2]R generation[0..12]: [161, 163, 163, 157, 163, 163, 160, 158, 163, 163, 157, 158]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307853046Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307866566Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307879326Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307900606Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 292","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307930726Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307945655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"":"downstairs","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307958635Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307970215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"0","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307983095Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"1","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.307998545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"client":"2","":"downstairs","session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.308011585Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.308023595Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"33225d26-ed40-4fee-86c3-db42153693c2 is now active with session: 51d923c0-7a03-4072-8513-66b62f91de3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.308036054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.308047824Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848,"session_id":"51d923c0-7a03-4072-8513-66b62f91de3e"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.308059614Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848} {"msg":"Activated sub_volume 33225d26-ed40-4fee-86c3-db42153693c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:23.308070544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 16384 data: 16 18 19 21 02/30 Read block 30 len 28672 03/30 Read block 473 len 40960 04/30 Read block 141 len 4096 05/30 Read block 77 len 20480 06/30 Read block 331 len 4096 07/30 Read block 470 len 16384 08/30 Read block 272 len 28672 09/30 Flush 10/30 Write block 260 len 32768 data: 9 7 10 11 16 16 13 13 11/30 Flush 12/30 Write block 12 len 8192 data: 14 15 13/30 Read block 318 len 32768 14/30 Write block 230 len 28672 data: 11 10 8 11 13 15 12 15/30 Write block 269 len 36864 data: 11 13 13 12 14 13 13 14 15 16/30 Write block 154 len 16384 data: 14 14 13 9 17/30 Read block 381 len 36864 18/30 Flush 19/30 Flush 20/30 Write block 572 len 24576 data: 12 10 12 14 13 17 21/30 Flush 22/30 Write block 444 len 20480 data: 16 15 15 17 15 23/30 Write block 215 len 36864 data: 11 15 15 16 16 19 23 23 21 24/30 Read block 250 len 36864 25/30 Flush 26/30 Write block 12 len 36864 data: 15 16 11 15 18 19 18 18 17 27/30 Read block 103 len 12288 28/30 Write block 238 len 20480 data: 12 12 16 14 12 29/30 Write block 221 len 20480 data: 24 24 22 19 13 30/30 Write block 53 len 24576 data: 13 14 13 13 9 6 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:23.328346111Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1848} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 294 294 294 T T T 1 020-039 163 163 163 291 291 291 T T T 2 040-059 163 163 163 291 291 291 T T T 3 060-079 157 157 157 280 280 280 F F F 4 080-099 163 163 163 291 291 291 F F F 5 100-119 163 163 163 291 291 291 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 164 164 164 294 294 294 F F F 8 160-179 163 163 163 291 291 291 F F F 9 180-199 163 163 163 291 291 291 F F F 10 200-219 164 164 164 297 297 297 F F F 11 220-239 164 164 164 297 297 297 T T T 12 240-259 163 163 163 291 291 291 T T T 13 260-279 164 164 164 294 294 294 F F F 14 280-299 160 160 160 287 287 287 F F F 15 300-319 161 161 161 288 288 288 F F F 16 320-339 162 162 162 290 290 290 F F F 17 340-359 163 163 163 291 291 291 F F F 18 360-379 160 160 160 285 285 285 F F F 19 380-399 164 164 164 292 292 292 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 163 163 163 291 291 291 F F F 22 440-459 164 164 164 297 297 297 F F F 23 460-479 162 162 162 290 290 290 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 160 160 160 287 287 287 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 162 162 162 289 289 289 F F F 28 560-579 164 164 164 296 296 296 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 164, Max flush: 297 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 298 298 298 F F F 1 020-039 165 165 165 298 298 298 F F F 2 040-059 165 165 165 298 298 298 F F F 3 060-079 157 157 157 280 280 280 F F F 4 080-099 163 163 163 291 291 291 F F F 5 100-119 163 163 163 291 291 291 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 164 164 164 294 294 294 F F F 8 160-179 163 163 163 291 291 291 F F F 9 180-199 163 163 163 291 291 291 F F F 10 200-219 164 164 164 297 297 297 F F F 11 220-239 165 165 165 298 298 298 F F F 12 240-259 165 165 165 298 298 298 F F F 13 260-279 164 164 164 294 294 294 F F F 14 280-299 160 160 160 287 287 287 F F F 15 300-319 161 161 161 288 288 288 F F F 16 320-339 162 162 162 290 290 290 F F F 17 340-359 163 163 163 291 291 291 F F F 18 360-379 160 160 160 285 285 285 F F F 19 380-399 164 164 164 292 292 292 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 163 163 163 291 291 291 F F F 22 440-459 164 164 164 297 297 297 F F F 23 460-479 162 162 162 290 290 290 F F F 24 480-499 155 155 155 277 277 277 F F F 25 500-519 160 160 160 287 287 287 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 162 162 162 289 289 289 F F F 28 560-579 164 164 164 296 296 296 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 165, Max flush: 298 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.57249487Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:27.615349202Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.615700629Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.647083295Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.64765394Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.6476893Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.64770334Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.647717989Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856} {"msg":"Crucible dae1f752-64a5-49c4-936d-a5f9becb7e39 has session id: af011c2b-3f32-4624-ac99-c9d2dbcd6310","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.648267935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Upstairs opts: Upstairs UUID: dae1f752-64a5-49c4-936d-a5f9becb7e39, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.648296294Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Crucible stats registered with UUID: dae1f752-64a5-49c4-936d-a5f9becb7e39","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.648311194Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:27.648323654Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.64871475Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.64875198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856} {"msg":"dae1f752-64a5-49c4-936d-a5f9becb7e39 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.64881489Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.648844799Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"0","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.648881319Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"0","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.648895919Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"1","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.648910619Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"1","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.648970198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"2","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.649087977Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"2","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.649133837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.649167596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.649384814Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"ds_connection connected from Ok(127.0.0.1:48342)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.649437124Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"ds_connection connected from Ok(127.0.0.1:34662)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.649477113Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"ds_connection connected from Ok(127.0.0.1:56327)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.649505153Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.650124438Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"1","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.650177627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"2","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.650408415Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"0","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651056659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"1","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651084209Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"2","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651099939Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"0","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651722793Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651748693Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[0]R flush_numbers[0..12]: [298, 298, 298, 280, 291, 291, 286, 294, 291, 291, 297, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651765673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[0]R generation[0..12]: [165, 165, 165, 157, 163, 163, 160, 164, 163, 163, 164, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651780263Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651794023Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[1]R flush_numbers[0..12]: [298, 298, 298, 280, 291, 291, 286, 294, 291, 291, 297, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651807543Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[1]R generation[0..12]: [165, 165, 165, 157, 163, 163, 160, 164, 163, 163, 164, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651820672Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651834062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[2]R flush_numbers[0..12]: [298, 298, 298, 280, 291, 291, 286, 294, 291, 291, 297, 298]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651847232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[2]R generation[0..12]: [165, 165, 165, 157, 163, 163, 160, 164, 163, 163, 164, 165]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651860642Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651874172Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651889892Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651904051Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651916702Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651934281Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"":"downstairs","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:48:27.651957841Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651977111Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"0","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.651990601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"1","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.652003621Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"client":"2","":"downstairs","session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.652020801Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.65204261Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"dae1f752-64a5-49c4-936d-a5f9becb7e39 is now active with session: af011c2b-3f32-4624-ac99-c9d2dbcd6310","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.65205807Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.65207005Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856,"session_id":"af011c2b-3f32-4624-ac99-c9d2dbcd6310"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.65208218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856} {"msg":"Activated sub_volume dae1f752-64a5-49c4-936d-a5f9becb7e39","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:27.65209378Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 463 len 4096 data: 11 03/30 Read block 114 len 16384 04/30 Flush 05/30 Write block 390 len 32768 data: 22 19 20 19 16 15 20 23 06/30 Read block 199 len 20480 07/30 Write block 45 len 28672 data: 12 10 10 10 10 11 13 08/30 Read block 3 len 20480 09/30 Write block 178 len 36864 data: 11 12 6 8 9 8 6 9 10 10/30 Write block 203 len 24576 data: 11 10 7 10 12 14 11/30 Write block 96 len 12288 data: 10 11 14 12/30 Read block 566 len 36864 13/30 Write block 42 len 20480 data: 13 17 15 13 11 14/30 Read block 544 len 20480 15/30 Read block 312 len 20480 16/30 Write block 495 len 24576 data: 13 9 12 10 9 12 17/30 Flush 18/30 Write block 112 len 32768 data: 12 12 11 9 11 12 14 15 19/30 Read block 161 len 32768 20/30 Write block 9 len 16384 data: 13 13 15 16 21/30 Read block 295 len 28672 22/30 Read block 345 len 40960 23/30 Read block 222 len 24576 24/30 Write block 340 len 36864 data: 16 16 11 9 9 12 10 10 11 25/30 Write block 388 len 8192 data: 19 20 26/30 Write block 235 len 32768 data: 16 13 10 13 13 17 15 13 27/30 Read block 262 len 24576 28/30 Write block 461 len 16384 data: 10 12 12 12 29/30 Read block 124 len 24576 30/30 Write block 95 len 36864 data: 12 11 12 15 15 17 17 14 16 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:27.674058461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1856} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 165 165 165 298 298 298 T T T 1 020-039 165 165 165 298 298 298 F F F 2 040-059 166 166 166 301 301 301 F F F 3 060-079 157 157 157 280 280 280 F F F 4 080-099 166 166 166 301 301 301 T T T 5 100-119 163 163 163 291 291 291 T T T 6 120-139 160 160 160 286 286 286 F F F 7 140-159 164 164 164 294 294 294 F F F 8 160-179 166 166 166 301 301 301 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 166 166 166 301 301 301 F F F 11 220-239 165 165 165 298 298 298 T T T 12 240-259 165 165 165 298 298 298 T T T 13 260-279 164 164 164 294 294 294 F F F 14 280-299 160 160 160 287 287 287 F F F 15 300-319 161 161 161 288 288 288 F F F 16 320-339 162 162 162 290 290 290 F F F 17 340-359 163 163 163 291 291 291 T T T 18 360-379 160 160 160 285 285 285 F F F 19 380-399 166 166 166 301 301 301 T T T 20 400-419 158 158 158 281 281 281 F F F 21 420-439 163 163 163 291 291 291 F F F 22 440-459 164 164 164 297 297 297 F F F 23 460-479 166 166 166 300 300 300 T T T 24 480-499 166 166 166 301 301 301 F F F 25 500-519 166 166 166 301 301 301 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 162 162 162 289 289 289 F F F 28 560-579 164 164 164 296 296 296 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 166, Max flush: 301 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 302 302 302 F F F 1 020-039 165 165 165 298 298 298 F F F 2 040-059 166 166 166 301 301 301 F F F 3 060-079 157 157 157 280 280 280 F F F 4 080-099 167 167 167 302 302 302 F F F 5 100-119 167 167 167 302 302 302 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 164 164 164 294 294 294 F F F 8 160-179 166 166 166 301 301 301 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 166 166 166 301 301 301 F F F 11 220-239 167 167 167 302 302 302 F F F 12 240-259 167 167 167 302 302 302 F F F 13 260-279 164 164 164 294 294 294 F F F 14 280-299 160 160 160 287 287 287 F F F 15 300-319 161 161 161 288 288 288 F F F 16 320-339 162 162 162 290 290 290 F F F 17 340-359 167 167 167 302 302 302 F F F 18 360-379 160 160 160 285 285 285 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 158 158 158 281 281 281 F F F 21 420-439 163 163 163 291 291 291 F F F 22 440-459 164 164 164 297 297 297 F F F 23 460-479 167 167 167 302 302 302 F F F 24 480-499 166 166 166 301 301 301 F F F 25 500-519 166 166 166 301 301 301 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 162 162 162 289 289 289 F F F 28 560-579 164 164 164 296 296 296 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 167, Max flush: 302 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:31.960049271Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.004448569Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.005088934Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.005133103Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.005147593Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.005161803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864} {"msg":"Crucible 2ba9de11-b59d-4620-8f58-0c9a77265e43 has session id: fa03efa1-f8a1-47cc-9698-bb548346cc99","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.005831277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Upstairs opts: Upstairs UUID: 2ba9de11-b59d-4620-8f58-0c9a77265e43, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.005872406Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Crucible stats registered with UUID: 2ba9de11-b59d-4620-8f58-0c9a77265e43","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.005894966Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:32.005908716Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006299123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006329972Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864} {"msg":"2ba9de11-b59d-4620-8f58-0c9a77265e43 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006369482Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006393962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"0","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006441601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"0","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006469901Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"1","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.00653356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"1","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.0065517Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"2","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.00656777Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"2","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.00658152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.00660046Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006627309Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"ds_connection connected from Ok(127.0.0.1:58607)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006654889Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"ds_connection connected from Ok(127.0.0.1:50814)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006683289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"ds_connection connected from Ok(127.0.0.1:47525)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.006717099Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.007312483Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"1","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.007342423Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"0","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.007501362Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"2","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.007986727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"1","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008031307Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"0","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008127826Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"2","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008567222Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008705531Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[0]R flush_numbers[0..12]: [302, 298, 301, 280, 302, 302, 286, 294, 301, 301, 301, 302]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.00880335Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[0]R generation[0..12]: [167, 165, 166, 157, 167, 167, 160, 164, 166, 166, 166, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.00882376Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.00884252Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[1]R flush_numbers[0..12]: [302, 298, 301, 280, 302, 302, 286, 294, 301, 301, 301, 302]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008856729Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[1]R generation[0..12]: [167, 165, 166, 157, 167, 167, 160, 164, 166, 166, 166, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008870619Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008885059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[2]R flush_numbers[0..12]: [302, 298, 301, 280, 302, 302, 286, 294, 301, 301, 301, 302]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008898859Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[2]R generation[0..12]: [167, 165, 166, 157, 167, 167, 160, 164, 166, 166, 166, 167]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008912539Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008926359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008939309Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008952268Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008964858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.008977658Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"":"downstairs","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009005248Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"Wait for a query_work_queue command to finish before sending IO session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009027868Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"0","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009041448Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"1","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009054447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"client":"2","":"downstairs","session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009069798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009082137Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"2ba9de11-b59d-4620-8f58-0c9a77265e43 is now active with session: fa03efa1-f8a1-47cc-9698-bb548346cc99","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009094877Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009106667Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864,"session_id":"fa03efa1-f8a1-47cc-9698-bb548346cc99"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009118667Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864} {"msg":"Activated sub_volume 2ba9de11-b59d-4620-8f58-0c9a77265e43","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:32.009129907Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 264 len 12288 data: 17 17 14 02/30 Read block 309 len 24576 03/30 Read block 103 len 4096 04/30 Write block 301 len 20480 data: 12 15 13 11 8 05/30 Flush 06/30 Write block 73 len 12288 data: 13 12 14 07/30 Read block 156 len 20480 08/30 Read block 382 len 8192 09/30 Flush 10/30 Flush 11/30 Write block 153 len 12288 data: 14 15 15 12/30 Read block 362 len 28672 13/30 Write block 321 len 16384 data: 16 14 13 12 14/30 Write block 551 len 24576 data: 16 15 16 13 13 14 15/30 Write block 330 len 28672 data: 17 17 15 14 16 17 16 16/30 Read block 333 len 20480 17/30 Read block 275 len 16384 18/30 Write block 272 len 40960 data: 13 15 14 14 15 16 18 17 19 19 19/30 Write block 310 len 28672 data: 12 11 11 11 9 8 11 20/30 Read block 70 len 40960 21/30 Read block 23 len 40960 22/30 Write block 411 len 24576 data: 12 13 12 10 12 13 23/30 Write block 34 len 40960 data: 10 9 11 12 11 12 14 15 14 18 24/30 Write block 223 len 16384 data: 23 20 14 12 25/30 Write block 342 len 16384 data: 12 10 10 13 26/30 Read block 553 len 32768 27/30 Read block 109 len 4096 28/30 Read block 266 len 20480 29/30 Read block 515 len 16384 30/30 Write block 0 len 24576 data: 4 6 6 8 9 10 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 3 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:32.028906768Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1864} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 167 167 167 302 302 302 T T T 1 020-039 165 165 165 298 298 298 T T T 2 040-059 166 166 166 301 301 301 T T T 3 060-079 168 168 168 304 304 304 F F F 4 080-099 167 167 167 302 302 302 F F F 5 100-119 167 167 167 302 302 302 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 164 164 164 294 294 294 T T T 8 160-179 166 166 166 301 301 301 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 166 166 166 301 301 301 F F F 11 220-239 167 167 167 302 302 302 T T T 12 240-259 167 167 167 302 302 302 F F F 13 260-279 168 168 168 303 303 303 T T T 14 280-299 160 160 160 287 287 287 T T T 15 300-319 168 168 168 303 303 303 T T T 16 320-339 162 162 162 290 290 290 T T T 17 340-359 167 167 167 302 302 302 T T T 18 360-379 160 160 160 285 285 285 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 158 158 158 281 281 281 T T T 21 420-439 163 163 163 291 291 291 F F F 22 440-459 164 164 164 297 297 297 F F F 23 460-479 167 167 167 302 302 302 F F F 24 480-499 166 166 166 301 301 301 F F F 25 500-519 166 166 166 301 301 301 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 162 162 162 289 289 289 T T T 28 560-579 164 164 164 296 296 296 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 168, Max flush: 304 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 169 169 169 305 305 305 F F F 1 020-039 169 169 169 305 305 305 F F F 2 040-059 169 169 169 305 305 305 F F F 3 060-079 168 168 168 304 304 304 F F F 4 080-099 167 167 167 302 302 302 F F F 5 100-119 167 167 167 302 302 302 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 169 169 169 305 305 305 F F F 8 160-179 166 166 166 301 301 301 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 166 166 166 301 301 301 F F F 11 220-239 169 169 169 305 305 305 F F F 12 240-259 167 167 167 302 302 302 F F F 13 260-279 169 169 169 305 305 305 F F F 14 280-299 169 169 169 305 305 305 F F F 15 300-319 169 169 169 305 305 305 F F F 16 320-339 169 169 169 305 305 305 F F F 17 340-359 169 169 169 305 305 305 F F F 18 360-379 160 160 160 285 285 285 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 163 163 163 291 291 291 F F F 22 440-459 164 164 164 297 297 297 F F F 23 460-479 167 167 167 302 302 302 F F F 24 480-499 166 166 166 301 301 301 F F F 25 500-519 166 166 166 301 301 301 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 169 169 169 305 305 305 F F F 28 560-579 164 164 164 296 296 296 F F F 29 580-599 158 158 158 281 281 281 F F F Max gen: 169, Max flush: 305 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.307831182Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.351527847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.352120682Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.352163251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.352178611Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.352192681Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872} {"msg":"Crucible 93e42ddc-a69e-4ffc-817c-8bbbcc3fdad7 has session id: ad6f7b23-630c-4924-9935-0cae39fd9384","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.352826035Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Upstairs opts: Upstairs UUID: 93e42ddc-a69e-4ffc-817c-8bbbcc3fdad7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.352861615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Crucible stats registered with UUID: 93e42ddc-a69e-4ffc-817c-8bbbcc3fdad7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.352876645Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:46.352889135Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353339411Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.35337258Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872} {"msg":"93e42ddc-a69e-4ffc-817c-8bbbcc3fdad7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.3534348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.35346233Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"0","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353492289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"0","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353506749Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"1","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353531339Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"1","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353557119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"2","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353585778Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"2","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353611798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353640618Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353667078Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"ds_connection connected from Ok(127.0.0.1:43449)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353694387Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"ds_connection connected from Ok(127.0.0.1:63079)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353723027Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"ds_connection connected from Ok(127.0.0.1:38722)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.353880466Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.354392421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"0","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.354422671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"1","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.355048455Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"1","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.355194104Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"0","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.355303273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"2","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.355699609Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.355735599Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.355963137Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"2","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[0]R flush_numbers[0..12]: [305, 305, 305, 304, 302, 302, 286, 305, 301, 301, 301, 305]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356486232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[0]R generation[0..12]: [169, 169, 169, 168, 167, 167, 160, 169, 166, 166, 166, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356506062Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356520852Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[1]R flush_numbers[0..12]: [305, 305, 305, 304, 302, 302, 286, 305, 301, 301, 301, 305]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356534822Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[1]R generation[0..12]: [169, 169, 169, 168, 167, 167, 160, 169, 166, 166, 166, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356549151Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356562782Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[2]R flush_numbers[0..12]: [305, 305, 305, 304, 302, 302, 286, 305, 301, 301, 301, 305]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356576171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[2]R generation[0..12]: [169, 169, 169, 168, 167, 167, 160, 169, 166, 166, 166, 169]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356589481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356606891Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356620721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356633571Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356646081Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356658741Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"":"downstairs","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356671481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.35668303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"0","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.35669601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"1","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.35670861Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"client":"2","":"downstairs","session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.3567215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.35674441Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"93e42ddc-a69e-4ffc-817c-8bbbcc3fdad7 is now active with session: ad6f7b23-630c-4924-9935-0cae39fd9384","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.3567589Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"Wait for a query_work_queue command to finish before sending IO ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.35677746Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872,"session_id":"ad6f7b23-630c-4924-9935-0cae39fd9384"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356789459Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872} {"msg":"Activated sub_volume 93e42ddc-a69e-4ffc-817c-8bbbcc3fdad7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:46.356800189Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 40 len 40960 data: 15 16 15 19 16 14 12 11 11 11 02/30 Read block 357 len 28672 03/30 Flush 04/30 Write block 16 len 20480 data: 19 20 19 19 18 05/30 Read block 402 len 4096 06/30 Read block 173 len 28672 07/30 Write block 353 len 40960 data: 14 7 8 10 13 13 13 13 10 9 08/30 Read block 267 len 32768 09/30 Flush 10/30 Write block 506 len 32768 data: 11 14 13 13 11 13 13 16 11/30 Read block 63 len 28672 12/30 Write block 496 len 4096 data: 10 13/30 Read block 588 len 20480 14/30 Read block 299 len 16384 15/30 Write block 168 len 16384 data: 7 8 8 7 16/30 Read block 538 len 8192 17/30 Flush 18/30 Write block 265 len 8192 data: 18 15 19/30 Write block 509 len 24576 data: 14 12 14 14 17 16 20/30 Write block 246 len 16384 data: 8 8 11 12 21/30 Read block 529 len 24576 22/30 Read block 252 len 32768 23/30 Write block 545 len 12288 data: 13 12 11 24/30 Flush 25/30 Flush 26/30 Write block 580 len 32768 data: 17 13 11 13 14 11 10 12 27/30 Read block 256 len 32768 28/30 Write block 238 len 32768 data: 14 14 18 16 14 10 9 9 29/30 Read block 435 len 16384 30/30 Write block 76 len 36864 data: 13 15 15 15 16 17 17 16 14 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:46.378341124Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1872} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 307 307 307 F F F 1 020-039 170 170 170 307 307 307 F F F 2 040-059 170 170 170 306 306 306 F F F 3 060-079 168 168 168 304 304 304 T T T 4 080-099 167 167 167 302 302 302 T T T 5 100-119 167 167 167 302 302 302 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 169 169 169 305 305 305 F F F 8 160-179 170 170 170 308 308 308 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 166 166 166 301 301 301 F F F 11 220-239 169 169 169 305 305 305 T T T 12 240-259 170 170 170 309 309 309 T T T 13 260-279 170 170 170 309 309 309 F F F 14 280-299 169 169 169 305 305 305 F F F 15 300-319 169 169 169 305 305 305 F F F 16 320-339 169 169 169 305 305 305 F F F 17 340-359 170 170 170 307 307 307 F F F 18 360-379 170 170 170 307 307 307 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 163 163 163 291 291 291 F F F 22 440-459 164 164 164 297 297 297 F F F 23 460-479 167 167 167 302 302 302 F F F 24 480-499 170 170 170 308 308 308 F F F 25 500-519 170 170 170 309 309 309 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 170 170 170 309 309 309 F F F 28 560-579 164 164 164 296 296 296 F F F 29 580-599 158 158 158 281 281 281 T T T Max gen: 170, Max flush: 309 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 307 307 307 F F F 1 020-039 170 170 170 307 307 307 F F F 2 040-059 170 170 170 306 306 306 F F F 3 060-079 171 171 171 310 310 310 F F F 4 080-099 171 171 171 310 310 310 F F F 5 100-119 167 167 167 302 302 302 F F F 6 120-139 160 160 160 286 286 286 F F F 7 140-159 169 169 169 305 305 305 F F F 8 160-179 170 170 170 308 308 308 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 166 166 166 301 301 301 F F F 11 220-239 171 171 171 310 310 310 F F F 12 240-259 171 171 171 310 310 310 F F F 13 260-279 170 170 170 309 309 309 F F F 14 280-299 169 169 169 305 305 305 F F F 15 300-319 169 169 169 305 305 305 F F F 16 320-339 169 169 169 305 305 305 F F F 17 340-359 170 170 170 307 307 307 F F F 18 360-379 170 170 170 307 307 307 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 163 163 163 291 291 291 F F F 22 440-459 164 164 164 297 297 297 F F F 23 460-479 167 167 167 302 302 302 F F F 24 480-499 170 170 170 308 308 308 F F F 25 500-519 170 170 170 309 309 309 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 170 170 170 309 309 309 F F F 28 560-579 164 164 164 296 296 296 F F F 29 580-599 171 171 171 310 310 310 F F F Max gen: 171, Max flush: 310 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.618324471Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.662918128Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.663718211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.66376387Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.66377909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.66379325Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880} {"msg":"Crucible 65ebf466-860a-4ad3-b7d4-b5b3ef4b7a16 has session id: 38e7cccb-9be6-40ec-bfb6-41a71e2a1661","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.664389934Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Upstairs opts: Upstairs UUID: 65ebf466-860a-4ad3-b7d4-b5b3ef4b7a16, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.664435474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Crucible stats registered with UUID: 65ebf466-860a-4ad3-b7d4-b5b3ef4b7a16","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.664459104Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:50.664495663Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.66490399Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.664937559Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880} {"msg":"65ebf466-860a-4ad3-b7d4-b5b3ef4b7a16 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.664985169Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665009289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"0","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665042279Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"0","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665056608Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"1","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665071748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"1","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665094418Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"2","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665118798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"2","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665140858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665163407Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665186507Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"ds_connection connected from Ok(127.0.0.1:42316)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665215047Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"ds_connection connected from Ok(127.0.0.1:61919)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665241007Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"ds_connection connected from Ok(127.0.0.1:61451)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.665269956Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.66593226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"0","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.66597229Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"2","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.666647094Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"2","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.666675534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"0","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.667220779Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"1","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.667248188Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.667276588Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.667875123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"1","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[0]R flush_numbers[0..12]: [307, 307, 306, 310, 310, 302, 286, 305, 308, 301, 301, 310]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668443868Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[0]R generation[0..12]: [170, 170, 170, 171, 171, 167, 160, 169, 170, 166, 166, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668476677Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668493117Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[1]R flush_numbers[0..12]: [307, 307, 306, 310, 310, 302, 286, 305, 308, 301, 301, 310]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668514827Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[1]R generation[0..12]: [170, 170, 170, 171, 171, 167, 160, 169, 170, 166, 166, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668539947Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668567156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[2]R flush_numbers[0..12]: [307, 307, 306, 310, 310, 302, 286, 305, 308, 301, 301, 310]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668592146Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[2]R generation[0..12]: [170, 170, 170, 171, 171, 167, 160, 169, 170, 166, 166, 171]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:48:50.668617876Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668649156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668672736Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668696405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668720795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668742405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"":"downstairs","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668755815Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668771435Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"0","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668794625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"1","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668818214Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"client":"2","":"downstairs","session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668842834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668859844Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"65ebf466-860a-4ad3-b7d4-b5b3ef4b7a16 is now active with session: 38e7cccb-9be6-40ec-bfb6-41a71e2a1661","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668872904Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668892784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880,"session_id":"38e7cccb-9be6-40ec-bfb6-41a71e2a1661"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668916633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880} {"msg":"Activated sub_volume 65ebf466-860a-4ad3-b7d4-b5b3ef4b7a16","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:50.668934893Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 567 len 40960 data: 14 13 14 11 13 13 11 13 15 14 02/30 Flush 03/30 Write block 502 len 8192 data: 13 15 04/30 Flush 05/30 Write block 134 len 40960 data: 16 16 15 16 14 12 12 12 11 10 06/30 Read block 334 len 24576 07/30 Write block 437 len 16384 data: 13 11 12 13 08/30 Read block 543 len 16384 09/30 Write block 270 len 32768 data: 14 14 14 16 15 15 16 17 10/30 Read block 205 len 32768 11/30 Write block 586 len 36864 data: 11 13 11 9 10 11 14 18 17 12/30 Read block 171 len 4096 13/30 Flush 14/30 Write block 78 len 40960 data: 16 16 17 18 18 17 15 12 15 13 15/30 Flush 16/30 Write block 18 len 32768 data: 20 20 19 12 13 10 10 11 17/30 Write block 56 len 12288 data: 14 10 7 18/30 Write block 341 len 36864 data: 17 13 11 11 14 11 11 12 11 19/30 Write block 554 len 40960 data: 14 14 15 15 10 11 10 10 9 11 20/30 Write block 258 len 32768 data: 11 8 10 8 11 12 18 19 21/30 Write block 548 len 4096 data: 12 22/30 Write block 362 len 20480 data: 10 10 9 7 8 23/30 Write block 142 len 20480 data: 12 11 12 12 11 24/30 Read block 470 len 8192 25/30 Write block 541 len 28672 data: 14 12 13 12 14 13 12 26/30 Read block 588 len 40960 27/30 Read block 274 len 20480 28/30 Write block 171 len 20480 data: 8 8 9 9 11 29/30 Write block 335 len 28672 data: 18 17 20 16 15 17 18 30/30 Write block 108 len 12288 data: 17 15 16 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:50.688815053Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1880} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 307 307 307 T T T 1 020-039 170 170 170 307 307 307 T T T 2 040-059 170 170 170 306 306 306 T T T 3 060-079 172 172 172 314 314 314 F F F 4 080-099 172 172 172 314 314 314 F F F 5 100-119 167 167 167 302 302 302 T T T 6 120-139 172 172 172 313 313 313 F F F 7 140-159 172 172 172 313 313 313 T T T 8 160-179 170 170 170 308 308 308 T T T 9 180-199 166 166 166 301 301 301 F F F 10 200-219 166 166 166 301 301 301 F F F 11 220-239 171 171 171 310 310 310 F F F 12 240-259 171 171 171 310 310 310 T T T 13 260-279 172 172 172 313 313 313 T T T 14 280-299 169 169 169 305 305 305 F F F 15 300-319 169 169 169 305 305 305 F F F 16 320-339 169 169 169 305 305 305 T T T 17 340-359 170 170 170 307 307 307 T T T 18 360-379 170 170 170 307 307 307 T T T 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 172 172 172 313 313 313 F F F 22 440-459 172 172 172 313 313 313 F F F 23 460-479 167 167 167 302 302 302 F F F 24 480-499 170 170 170 308 308 308 F F F 25 500-519 172 172 172 312 312 312 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 170 170 170 309 309 309 T T T 28 560-579 172 172 172 311 311 311 T T T 29 580-599 172 172 172 313 313 313 F F F Max gen: 172, Max flush: 314 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 315 315 315 F F F 1 020-039 173 173 173 315 315 315 F F F 2 040-059 173 173 173 315 315 315 F F F 3 060-079 172 172 172 314 314 314 F F F 4 080-099 172 172 172 314 314 314 F F F 5 100-119 173 173 173 315 315 315 F F F 6 120-139 172 172 172 313 313 313 F F F 7 140-159 173 173 173 315 315 315 F F F 8 160-179 173 173 173 315 315 315 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 166 166 166 301 301 301 F F F 11 220-239 171 171 171 310 310 310 F F F 12 240-259 173 173 173 315 315 315 F F F 13 260-279 173 173 173 315 315 315 F F F 14 280-299 169 169 169 305 305 305 F F F 15 300-319 169 169 169 305 305 305 F F F 16 320-339 173 173 173 315 315 315 F F F 17 340-359 173 173 173 315 315 315 F F F 18 360-379 173 173 173 315 315 315 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 172 172 172 313 313 313 F F F 22 440-459 172 172 172 313 313 313 F F F 23 460-479 167 167 167 302 302 302 F F F 24 480-499 170 170 170 308 308 308 F F F 25 500-519 172 172 172 312 312 312 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 173 173 173 315 315 315 F F F 28 560-579 173 173 173 315 315 315 F F F 29 580-599 172 172 172 313 313 313 F F F Max gen: 173, Max flush: 315 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:54.973160548Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.017500927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.018170771Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.018216101Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.018232171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.01824722Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888} {"msg":"Crucible 69676cc2-0c5a-4cea-aa9e-10ecf61b56cf has session id: c6fa554a-254e-4245-8e64-53c0ed0f6bd6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.018808615Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Upstairs opts: Upstairs UUID: 69676cc2-0c5a-4cea-aa9e-10ecf61b56cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.018847165Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Crucible stats registered with UUID: 69676cc2-0c5a-4cea-aa9e-10ecf61b56cf","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.018869805Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:55.018890055Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019335001Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.01936224Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888} {"msg":"69676cc2-0c5a-4cea-aa9e-10ecf61b56cf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.01944794Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019467289Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"0","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019501799Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"0","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019516099Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"1","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019530259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"1","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019543629Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"2","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019557488Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"2","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019570259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019583918Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019606998Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46197)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019693027Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35546)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019730737Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:50955)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.019757437Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.020501Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"2","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.02054458Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"0","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.020946786Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"1","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.021107405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"0","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.021231724Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"2","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.021514171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"1","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.021762329Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.021878707Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[0]R flush_numbers[0..12]: [315, 315, 315, 314, 314, 315, 313, 315, 315, 301, 301, 310]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022203705Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[0]R generation[0..12]: [173, 173, 173, 172, 172, 173, 172, 173, 173, 166, 166, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022232574Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022248024Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[1]R flush_numbers[0..12]: [315, 315, 315, 314, 314, 315, 313, 315, 315, 301, 301, 310]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022262324Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[1]R generation[0..12]: [173, 173, 173, 172, 172, 173, 172, 173, 173, 166, 166, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022276444Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022290394Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[2]R flush_numbers[0..12]: [315, 315, 315, 314, 314, 315, 313, 315, 315, 301, 301, 310]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022303864Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[2]R generation[0..12]: [173, 173, 173, 172, 172, 173, 172, 173, 173, 166, 166, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022317234Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022330803Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022347283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:48:55.022360673Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022376633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022389183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"":"downstairs","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022402033Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022413823Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"0","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022426853Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"1","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022439813Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"client":"2","":"downstairs","session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022452632Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022464622Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"69676cc2-0c5a-4cea-aa9e-10ecf61b56cf is now active with session: c6fa554a-254e-4245-8e64-53c0ed0f6bd6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022477152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022488962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888,"session_id":"c6fa554a-254e-4245-8e64-53c0ed0f6bd6"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022501112Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888} {"msg":"Activated sub_volume 69676cc2-0c5a-4cea-aa9e-10ecf61b56cf","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:55.022512192Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 11 len 28672 data: 16 17 17 12 16 20 21 02/30 Read block 301 len 24576 03/30 Write block 207 len 40960 data: 13 15 16 13 11 10 11 12 12 16 04/30 Read block 304 len 12288 05/30 Read block 442 len 40960 06/30 Write block 108 len 24576 data: 18 16 17 14 13 13 07/30 Write block 479 len 28672 data: 8 9 9 6 8 8 9 08/30 Read block 217 len 4096 09/30 Write block 62 len 36864 data: 16 14 15 18 20 17 18 18 14 10/30 Read block 58 len 32768 11/30 Read block 116 len 24576 12/30 Read block 185 len 24576 13/30 Write block 592 len 4096 data: 15 14/30 Write block 341 len 40960 data: 19 14 12 12 15 12 12 13 12 17 15/30 Read block 456 len 8192 16/30 Write block 345 len 28672 data: 16 13 13 14 13 18 17 17/30 Read block 244 len 24576 18/30 Read block 243 len 24576 19/30 Write block 249 len 24576 data: 13 11 16 17 18 17 20/30 Write block 569 len 4096 data: 15 21/30 Write block 30 len 24576 data: 15 13 13 11 11 10 22/30 Read block 41 len 32768 23/30 Flush 24/30 Write block 227 len 20480 data: 14 13 14 12 11 25/30 Write block 117 len 32768 data: 13 15 16 21 20 19 17 16 26/30 Read block 487 len 36864 27/30 Write block 292 len 36864 data: 16 16 16 13 10 10 9 8 10 28/30 Read block 385 len 36864 29/30 Write block 511 len 8192 data: 15 15 30/30 Write block 8 len 20480 data: 15 14 14 17 18 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:55.044188166Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1888} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 316 316 316 T T T 1 020-039 174 174 174 316 316 316 F F F 2 040-059 173 173 173 315 315 315 F F F 3 060-079 174 174 174 316 316 316 F F F 4 080-099 172 172 172 314 314 314 F F F 5 100-119 174 174 174 316 316 316 T T T 6 120-139 172 172 172 313 313 313 T T T 7 140-159 173 173 173 315 315 315 F F F 8 160-179 173 173 173 315 315 315 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 171 171 171 310 310 310 T T T 12 240-259 174 174 174 316 316 316 F F F 13 260-279 173 173 173 315 315 315 F F F 14 280-299 169 169 169 305 305 305 T T T 15 300-319 169 169 169 305 305 305 T T T 16 320-339 173 173 173 315 315 315 F F F 17 340-359 174 174 174 316 316 316 F F F 18 360-379 173 173 173 315 315 315 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 172 172 172 313 313 313 F F F 22 440-459 172 172 172 313 313 313 F F F 23 460-479 174 174 174 316 316 316 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 172 172 172 312 312 312 T T T 26 520-539 160 160 160 287 287 287 F F F 27 540-559 173 173 173 315 315 315 F F F 28 560-579 174 174 174 316 316 316 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 174, Max flush: 316 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 317 317 317 F F F 1 020-039 174 174 174 316 316 316 F F F 2 040-059 173 173 173 315 315 315 F F F 3 060-079 174 174 174 316 316 316 F F F 4 080-099 172 172 172 314 314 314 F F F 5 100-119 175 175 175 317 317 317 F F F 6 120-139 175 175 175 317 317 317 F F F 7 140-159 173 173 173 315 315 315 F F F 8 160-179 173 173 173 315 315 315 F F F 9 180-199 166 166 166 301 301 301 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 175 175 175 317 317 317 F F F 12 240-259 174 174 174 316 316 316 F F F 13 260-279 173 173 173 315 315 315 F F F 14 280-299 175 175 175 317 317 317 F F F 15 300-319 175 175 175 317 317 317 F F F 16 320-339 173 173 173 315 315 315 F F F 17 340-359 174 174 174 316 316 316 F F F 18 360-379 173 173 173 315 315 315 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 172 172 172 313 313 313 F F F 22 440-459 172 172 172 313 313 313 F F F 23 460-479 174 174 174 316 316 316 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 160 160 160 287 287 287 F F F 27 540-559 173 173 173 315 315 315 F F F 28 560-579 174 174 174 316 316 316 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 175, Max flush: 317 Begin loop 88 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.298437694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.342253697Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.342857172Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.342899211Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.342913831Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.342928181Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896} {"msg":"Crucible 772bc7c6-206b-4029-a325-fb31da3451c2 has session id: 7b06c8e7-9917-4d19-a95b-e4c4062305ce","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.343422546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Upstairs opts: Upstairs UUID: 772bc7c6-206b-4029-a325-fb31da3451c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.343466786Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Crucible stats registered with UUID: 772bc7c6-206b-4029-a325-fb31da3451c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.343490356Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:59.343511776Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344013681Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344071481Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896} {"msg":"772bc7c6-206b-4029-a325-fb31da3451c2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.34412869Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.34416303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"0","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344211739Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"0","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344235859Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"1","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344300079Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"1","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344339468Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"2","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344368268Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"2","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344392348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344417058Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344443067Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:38407)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344475597Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:62623)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344586436Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:39187)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.344625286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.345148501Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"2","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.345177631Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"0","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.34529184Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"1","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.345833685Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"2","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.345861944Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"0","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.345905824Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"1","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346449159Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346549258Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[0]R flush_numbers[0..12]: [317, 316, 315, 316, 314, 317, 317, 315, 315, 301, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346623488Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[0]R generation[0..12]: [175, 174, 173, 174, 172, 175, 175, 173, 173, 166, 174, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346643328Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346657527Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[1]R flush_numbers[0..12]: [317, 316, 315, 316, 314, 317, 317, 315, 315, 301, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346675627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[1]R generation[0..12]: [175, 174, 173, 174, 172, 175, 175, 173, 173, 166, 174, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346689537Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346703297Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[2]R flush_numbers[0..12]: [317, 316, 315, 316, 314, 317, 317, 315, 315, 301, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346717337Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[2]R generation[0..12]: [175, 174, 173, 174, 172, 175, 175, 173, 173, 166, 174, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346731187Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346744796Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346757547Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346770546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346783026Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346795666Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"":"downstairs","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346808686Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346820276Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"0","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346833616Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"1","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346846576Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"client":"2","":"downstairs","session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346859546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346873175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"772bc7c6-206b-4029-a325-fb31da3451c2 is now active with session: 7b06c8e7-9917-4d19-a95b-e4c4062305ce","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:48:59.346885875Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346900815Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896,"session_id":"7b06c8e7-9917-4d19-a95b-e4c4062305ce"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346912925Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896} {"msg":"Activated sub_volume 772bc7c6-206b-4029-a325-fb31da3451c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:48:59.346923795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 137 len 32768 data: 17 15 13 13 13 13 12 13 02/30 Flush 03/30 Write block 430 len 24576 data: 11 10 9 10 10 13 04/30 Read block 454 len 28672 05/30 Read block 316 len 24576 06/30 Read block 284 len 40960 07/30 Read block 379 len 24576 08/30 Write block 12 len 24576 data: 19 18 13 17 21 22 09/30 Read block 162 len 20480 10/30 Read block 197 len 24576 11/30 Read block 37 len 16384 12/30 Read block 538 len 12288 13/30 Flush 14/30 Write block 468 len 16384 data: 17 12 12 15 15/30 Flush 16/30 Write block 89 len 24576 data: 19 17 17 15 13 11 17/30 Flush 18/30 Write block 366 len 40960 data: 9 7 7 10 10 12 16 19 19 19 19/30 Write block 270 len 4096 data: 15 20/30 Read block 466 len 16384 21/30 Write block 187 len 24576 data: 11 10 13 12 11 11 22/30 Read block 325 len 32768 23/30 Flush 24/30 Write block 165 len 28672 data: 4 5 5 8 9 9 9 25/30 Read block 200 len 8192 26/30 Read block 85 len 28672 27/30 Read block 316 len 20480 28/30 Write block 26 len 8192 data: 14 16 29/30 Write block 319 len 24576 data: 15 16 17 15 14 13 30/30 Write block 530 len 32768 data: 15 12 13 13 14 16 13 11 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:48:59.367866156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1896} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 319 319 319 F F F 1 020-039 174 174 174 316 316 316 T T T 2 040-059 173 173 173 315 315 315 F F F 3 060-079 174 174 174 316 316 316 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 175 175 175 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 173 173 173 315 315 315 T T T 9 180-199 176 176 176 322 322 322 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 175 175 175 317 317 317 F F F 12 240-259 174 174 174 316 316 316 F F F 13 260-279 176 176 176 322 322 322 F F F 14 280-299 175 175 175 317 317 317 F F F 15 300-319 175 175 175 317 317 317 T T T 16 320-339 173 173 173 315 315 315 T T T 17 340-359 174 174 174 316 316 316 F F F 18 360-379 176 176 176 322 322 322 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 176 176 176 319 319 319 F F F 22 440-459 172 172 172 313 313 313 F F F 23 460-479 176 176 176 320 320 320 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 160 160 160 287 287 287 T T T 27 540-559 173 173 173 315 315 315 F F F 28 560-579 174 174 174 316 316 316 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 176, Max flush: 322 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 319 319 319 F F F 1 020-039 177 177 177 323 323 323 F F F 2 040-059 173 173 173 315 315 315 F F F 3 060-079 174 174 174 316 316 316 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 175 175 175 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 177 177 177 323 323 323 F F F 9 180-199 176 176 176 322 322 322 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 175 175 175 317 317 317 F F F 12 240-259 174 174 174 316 316 316 F F F 13 260-279 176 176 176 322 322 322 F F F 14 280-299 175 175 175 317 317 317 F F F 15 300-319 177 177 177 323 323 323 F F F 16 320-339 177 177 177 323 323 323 F F F 17 340-359 174 174 174 316 316 316 F F F 18 360-379 176 176 176 322 322 322 F F F 19 380-399 167 167 167 302 302 302 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 176 176 176 319 319 319 F F F 22 440-459 172 172 172 313 313 313 F F F 23 460-479 176 176 176 320 320 320 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 177 177 177 323 323 323 F F F 27 540-559 173 173 173 315 315 315 F F F 28 560-579 174 174 174 316 316 316 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 177, Max flush: 323 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.605681581Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.650507296Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.651219749Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.651276449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.651303059Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.651327279Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904} {"msg":"Crucible 1f414785-d2d9-4eca-82f3-64855d2cf1fb has session id: 734bd310-c69a-4f93-af5d-2a5b583d0f77","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.651962113Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Upstairs opts: Upstairs UUID: 1f414785-d2d9-4eca-82f3-64855d2cf1fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652013722Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Crucible stats registered with UUID: 1f414785-d2d9-4eca-82f3-64855d2cf1fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652044002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:03.652070012Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652490148Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652525528Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904} {"msg":"1f414785-d2d9-4eca-82f3-64855d2cf1fb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652584487Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652610847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"0","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652640017Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"0","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652654236Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"1","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652676956Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"1","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652702796Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"2","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652726756Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"2","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652750646Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652773586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652793055Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"ds_connection connected from Ok(127.0.0.1:34425)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652810845Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"ds_connection connected from Ok(127.0.0.1:36133)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652833325Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"ds_connection connected from Ok(127.0.0.1:40793)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.652858125Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.653509939Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"1","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.653537628Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"0","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.654076314Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"2","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.654230232Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"1","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.654267852Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"0","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.654635509Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"2","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.654795127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.654813897Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[0]R flush_numbers[0..12]: [319, 323, 315, 316, 321, 317, 318, 318, 323, 322, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655207813Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[0]R generation[0..12]: [176, 177, 173, 174, 176, 175, 176, 176, 177, 176, 174, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655233453Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655248773Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[1]R flush_numbers[0..12]: [319, 323, 315, 316, 321, 317, 318, 318, 323, 322, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655263113Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[1]R generation[0..12]: [176, 177, 173, 174, 176, 175, 176, 176, 177, 176, 174, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655276823Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655290563Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[2]R flush_numbers[0..12]: [319, 323, 315, 316, 321, 317, 318, 318, 323, 322, 316, 317]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655304332Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[2]R generation[0..12]: [176, 177, 173, 174, 176, 175, 176, 176, 177, 176, 174, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655317932Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655331672Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655344472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655357272Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655379322Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655391802Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"":"downstairs","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655405052Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655416762Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"0","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655430031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"1","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655443331Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"client":"2","":"downstairs","session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655457001Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655468861Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"1f414785-d2d9-4eca-82f3-64855d2cf1fb is now active with session: 734bd310-c69a-4f93-af5d-2a5b583d0f77","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655481361Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655492981Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904,"session_id":"734bd310-c69a-4f93-af5d-2a5b583d0f77"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655505031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904} {"msg":"Activated sub_volume 1f414785-d2d9-4eca-82f3-64855d2cf1fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:03.655515991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 193 len 4096 data: 9 02/30 Read block 207 len 36864 03/30 Write block 34 len 16384 data: 12 11 12 13 04/30 Read block 308 len 20480 05/30 Write block 283 len 40960 data: 17 16 11 12 10 12 14 17 16 17 06/30 Read block 405 len 32768 07/30 Write block 244 len 40960 data: 10 10 9 9 12 14 12 17 18 19 08/30 Read block 206 len 24576 09/30 Flush 10/30 Write block 454 len 32768 data: 10 11 11 10 12 10 10 11 11/30 Read block 167 len 20480 12/30 Read block 29 len 28672 13/30 Write block 269 len 12288 data: 12 16 15 14/30 Read block 395 len 20480 15/30 Write block 383 len 32768 data: 14 14 14 13 17 20 21 23 16/30 Read block 361 len 16384 17/30 Read block 179 len 20480 18/30 Write block 306 len 32768 data: 8 9 11 14 13 12 12 12 19/30 Read block 417 len 4096 20/30 Read block 189 len 40960 21/30 Read block 591 len 8192 22/30 Read block 260 len 36864 23/30 Read block 385 len 4096 24/30 Read block 316 len 28672 25/30 Read block 41 len 36864 26/30 Read block 293 len 36864 27/30 Read block 26 len 8192 28/30 Write block 520 len 40960 data: 13 16 15 12 15 14 15 15 13 16 29/30 Write block 238 len 32768 data: 15 15 19 17 15 11 11 11 30/30 Read block 396 len 36864 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 8 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 8 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 19 21 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:03.678193805Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1904} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 319 319 319 F F F 1 020-039 178 178 178 324 324 324 F F F 2 040-059 173 173 173 315 315 315 F F F 3 060-079 174 174 174 316 316 316 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 175 175 175 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 177 177 177 323 323 323 F F F 9 180-199 178 178 178 324 324 324 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 175 175 175 317 317 317 T T T 12 240-259 178 178 178 324 324 324 T T T 13 260-279 176 176 176 322 322 322 T T T 14 280-299 178 178 178 324 324 324 F F F 15 300-319 177 177 177 323 323 323 T T T 16 320-339 177 177 177 323 323 323 F F F 17 340-359 174 174 174 316 316 316 F F F 18 360-379 176 176 176 322 322 322 F F F 19 380-399 167 167 167 302 302 302 T T T 20 400-419 169 169 169 305 305 305 F F F 21 420-439 176 176 176 319 319 319 F F F 22 440-459 172 172 172 313 313 313 T T T 23 460-479 176 176 176 320 320 320 T T T 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 177 177 177 323 323 323 T T T 27 540-559 173 173 173 315 315 315 F F F 28 560-579 174 174 174 316 316 316 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 178, Max flush: 324 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 319 319 319 F F F 1 020-039 178 178 178 324 324 324 F F F 2 040-059 173 173 173 315 315 315 F F F 3 060-079 174 174 174 316 316 316 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 175 175 175 317 317 317 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 177 177 177 323 323 323 F F F 9 180-199 178 178 178 324 324 324 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 179 179 179 325 325 325 F F F 12 240-259 179 179 179 325 325 325 F F F 13 260-279 179 179 179 325 325 325 F F F 14 280-299 178 178 178 324 324 324 F F F 15 300-319 179 179 179 325 325 325 F F F 16 320-339 177 177 177 323 323 323 F F F 17 340-359 174 174 174 316 316 316 F F F 18 360-379 176 176 176 322 322 322 F F F 19 380-399 179 179 179 325 325 325 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 176 176 176 319 319 319 F F F 22 440-459 179 179 179 325 325 325 F F F 23 460-479 179 179 179 325 325 325 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 179 179 179 325 325 325 F F F 27 540-559 173 173 173 315 315 315 F F F 28 560-579 174 174 174 316 316 316 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 179, Max flush: 325 Begin loop 90 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:17.937522307Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:17.981200502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:17.981755946Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.01337632Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.014048624Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.014083924Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.014097854Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.014111924Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912} {"msg":"Crucible b37dd716-8275-42bf-887f-c812659bc533 has session id: fe332582-0884-46b5-8d6b-61ed67a1e1ad","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.014716368Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Upstairs opts: Upstairs UUID: b37dd716-8275-42bf-887f-c812659bc533, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.014752268Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Crucible stats registered with UUID: b37dd716-8275-42bf-887f-c812659bc533","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.014770058Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:18.014785798Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015214824Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015244283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912} {"msg":"b37dd716-8275-42bf-887f-c812659bc533 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015312283Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015333773Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"0","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015369682Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"0","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015387162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"1","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015404682Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"1","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015420622Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"2","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015438462Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"2","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015454502Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015476572Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015490421Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:51292)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015504171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:56274)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015545891Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:55837)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.015783659Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.016297404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"1","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.016343674Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"2","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.016550732Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"0","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017009118Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"2","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017057997Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"1","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017199346Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"0","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017645902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017760851Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[0]R flush_numbers[0..12]: [319, 324, 315, 316, 321, 317, 318, 318, 323, 324, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.01788554Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[0]R generation[0..12]: [176, 178, 173, 174, 176, 175, 176, 176, 177, 178, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.01790621Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017925209Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[1]R flush_numbers[0..12]: [319, 324, 315, 316, 321, 317, 318, 318, 323, 324, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017940009Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[1]R generation[0..12]: [176, 178, 173, 174, 176, 175, 176, 176, 177, 178, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017953679Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017967609Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[2]R flush_numbers[0..12]: [319, 324, 315, 316, 321, 317, 318, 318, 323, 324, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.017981269Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[2]R generation[0..12]: [176, 178, 173, 174, 176, 175, 176, 176, 177, 178, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018009309Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018024789Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018038438Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018058148Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018071618Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018084388Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"":"downstairs","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018097068Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018108928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"0","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018122217Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"1","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018137837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"client":"2","":"downstairs","session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:49:18.018150967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018168277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"b37dd716-8275-42bf-887f-c812659bc533 is now active with session: fe332582-0884-46b5-8d6b-61ed67a1e1ad","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018181057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018192887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912,"session_id":"fe332582-0884-46b5-8d6b-61ed67a1e1ad"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018204887Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912} {"msg":"Activated sub_volume b37dd716-8275-42bf-887f-c812659bc533","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:18.018215857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 104 len 12288 data: 12 16 20 02/30 Write block 173 len 36864 data: 10 10 12 11 10 12 13 7 9 03/30 Flush 04/30 Write block 4 len 4096 data: 10 05/30 Read block 563 len 4096 06/30 Read block 82 len 4096 07/30 Read block 355 len 12288 08/30 Read block 316 len 8192 09/30 Read block 589 len 8192 10/30 Write block 535 len 40960 data: 17 14 12 13 14 13 15 13 14 13 11/30 Read block 404 len 12288 12/30 Write block 325 len 32768 data: 14 10 14 15 16 18 18 16 13/30 Read block 0 len 16384 14/30 Read block 471 len 12288 15/30 Read block 249 len 8192 16/30 Write block 255 len 32768 data: 17 15 13 12 9 11 9 12 17/30 Write block 267 len 36864 data: 14 11 13 17 16 15 17 16 16 18/30 Read block 581 len 24576 19/30 Read block 160 len 20480 20/30 Write block 549 len 24576 data: 14 12 17 16 17 15 21/30 Read block 578 len 24576 22/30 Flush 23/30 Write block 16 len 32768 data: 22 23 21 21 20 13 14 11 24/30 Read block 590 len 12288 25/30 Read block 201 len 24576 26/30 Write block 355 len 24576 data: 9 11 14 14 14 14 27/30 Read block 549 len 4096 28/30 Write block 22 len 40960 data: 15 12 11 12 15 17 19 16 16 14 29/30 Write block 64 len 16384 data: 16 19 21 18 30/30 Read block 391 len 36864 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 7 8 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:18.039525624Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1912} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 327 327 327 T T T 1 020-039 178 178 178 324 324 324 T T T 2 040-059 173 173 173 315 315 315 F F F 3 060-079 174 174 174 316 316 316 T T T 4 080-099 176 176 176 321 321 321 F F F 5 100-119 180 180 180 326 326 326 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 180 180 180 326 326 326 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 179 179 179 325 325 325 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 178 178 178 324 324 324 F F F 15 300-319 179 179 179 325 325 325 F F F 16 320-339 180 180 180 327 327 327 F F F 17 340-359 174 174 174 316 316 316 T T T 18 360-379 176 176 176 322 322 322 T T T 19 380-399 179 179 179 325 325 325 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 176 176 176 319 319 319 F F F 22 440-459 179 179 179 325 325 325 F F F 23 460-479 179 179 179 325 325 325 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 180 180 180 327 327 327 F F F 27 540-559 180 180 180 327 327 327 F F F 28 560-579 174 174 174 316 316 316 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 180, Max flush: 327 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 328 328 328 F F F 1 020-039 181 181 181 328 328 328 F F F 2 040-059 173 173 173 315 315 315 F F F 3 060-079 181 181 181 328 328 328 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 180 180 180 326 326 326 F F F 6 120-139 176 176 176 318 318 318 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 180 180 180 326 326 326 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 179 179 179 325 325 325 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 178 178 178 324 324 324 F F F 15 300-319 179 179 179 325 325 325 F F F 16 320-339 180 180 180 327 327 327 F F F 17 340-359 181 181 181 328 328 328 F F F 18 360-379 181 181 181 328 328 328 F F F 19 380-399 179 179 179 325 325 325 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 176 176 176 319 319 319 F F F 22 440-459 179 179 179 325 325 325 F F F 23 460-479 179 179 179 325 325 325 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 180 180 180 327 327 327 F F F 27 540-559 180 180 180 327 327 327 F F F 28 560-579 174 174 174 316 316 316 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 181, Max flush: 328 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.318167491Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:22.361839496Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.362215552Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.394162614Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.394786408Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.394818508Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.394831888Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.394845987Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920} {"msg":"Crucible d879a312-52a5-41f7-8ade-ead4cca52e9c has session id: e7560078-b7c8-4978-9786-e0a29d2114b7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.395522511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Upstairs opts: Upstairs UUID: d879a312-52a5-41f7-8ade-ead4cca52e9c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.395552841Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Crucible stats registered with UUID: d879a312-52a5-41f7-8ade-ead4cca52e9c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.395568241Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:22.395580981Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.395984977Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396015167Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920} {"msg":"d879a312-52a5-41f7-8ade-ead4cca52e9c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396092526Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396110506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"0","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396140596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"0","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396155436Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"1","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396170365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"1","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396183795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"2","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396197965Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"2","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396210675Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396224175Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396237735Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53132)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396417213Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44847)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396535602Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49726)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.396577102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.397326825Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"1","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.397355925Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"2","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.397376534Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"0","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398036608Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"1","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398065198Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"0","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398081958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"2","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398669143Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398776002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[0]R flush_numbers[0..12]: [328, 328, 315, 328, 321, 326, 318, 318, 326, 326, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398849541Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[0]R generation[0..12]: [181, 181, 173, 181, 176, 180, 176, 176, 180, 180, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398868911Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398883581Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[1]R flush_numbers[0..12]: [328, 328, 315, 328, 321, 326, 318, 318, 326, 326, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398897381Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[1]R generation[0..12]: [181, 181, 173, 181, 176, 180, 176, 176, 180, 180, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.398911031Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.39892476Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[2]R flush_numbers[0..12]: [328, 328, 315, 328, 321, 326, 318, 318, 326, 326, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.39893857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[2]R generation[0..12]: [181, 181, 173, 181, 176, 180, 176, 176, 180, 180, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.39895196Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.39896916Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.39898208Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.39899484Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.39900748Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399020199Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"":"downstairs","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399039509Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399051609Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"0","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399064589Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"1","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399077409Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"client":"2","":"downstairs","session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399090129Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399102119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"d879a312-52a5-41f7-8ade-ead4cca52e9c is now active with session: e7560078-b7c8-4978-9786-e0a29d2114b7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399114719Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399126718Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920,"session_id":"e7560078-b7c8-4978-9786-e0a29d2114b7"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399138878Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920} {"msg":"Activated sub_volume d879a312-52a5-41f7-8ade-ead4cca52e9c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:22.399149948Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 362 len 12288 data: 11 11 10 03/30 Write block 387 len 20480 data: 18 21 22 24 20 04/30 Read block 323 len 12288 05/30 Read block 543 len 20480 06/30 Read block 177 len 20480 07/30 Read block 364 len 20480 08/30 Write block 130 len 32768 data: 13 14 16 16 17 17 16 18 09/30 Write block 341 len 16384 data: 20 15 13 13 10/30 Write block 123 len 16384 data: 18 17 14 14 11/30 Read block 329 len 4096 12/30 Write block 42 len 16384 data: 16 20 17 15 13/30 Read block 394 len 32768 14/30 Write block 354 len 12288 data: 8 10 12 15/30 Flush 16/30 Write block 33 len 24576 data: 12 13 12 13 14 12 17/30 Read block 265 len 4096 18/30 Read block 239 len 40960 19/30 Write block 51 len 32768 data: 14 16 14 15 14 15 11 8 20/30 Write block 34 len 12288 data: 14 13 14 21/30 Read block 253 len 24576 22/30 Read block 526 len 16384 23/30 Read block 255 len 8192 24/30 Read block 229 len 24576 25/30 Read block 343 len 32768 26/30 Write block 555 len 24576 data: 15 16 16 11 12 11 27/30 Write block 262 len 12288 data: 13 13 19 28/30 Read block 197 len 40960 29/30 Read block 99 len 40960 30/30 Read block 340 len 12288 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 6 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 15 14 14 43 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:22.419143737Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1920} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 328 328 328 F F F 1 020-039 181 181 181 328 328 328 T T T 2 040-059 182 182 182 330 330 330 T T T 3 060-079 181 181 181 328 328 328 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 180 180 180 326 326 326 F F F 6 120-139 182 182 182 330 330 330 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 180 180 180 326 326 326 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 179 179 179 325 325 325 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 180 180 180 327 327 327 T T T 14 280-299 178 178 178 324 324 324 F F F 15 300-319 179 179 179 325 325 325 F F F 16 320-339 180 180 180 327 327 327 F F F 17 340-359 182 182 182 330 330 330 F F F 18 360-379 182 182 182 330 330 330 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 176 176 176 319 319 319 F F F 22 440-459 179 179 179 325 325 325 F F F 23 460-479 179 179 179 325 325 325 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 180 180 180 327 327 327 F F F 27 540-559 180 180 180 327 327 327 T T T 28 560-579 174 174 174 316 316 316 T T T 29 580-599 174 174 174 316 316 316 F F F Max gen: 182, Max flush: 330 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 328 328 328 F F F 1 020-039 183 183 183 331 331 331 F F F 2 040-059 183 183 183 331 331 331 F F F 3 060-079 181 181 181 328 328 328 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 180 180 180 326 326 326 F F F 6 120-139 182 182 182 330 330 330 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 180 180 180 326 326 326 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 174 174 174 316 316 316 F F F 11 220-239 179 179 179 325 325 325 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 183 183 183 331 331 331 F F F 14 280-299 178 178 178 324 324 324 F F F 15 300-319 179 179 179 325 325 325 F F F 16 320-339 180 180 180 327 327 327 F F F 17 340-359 182 182 182 330 330 330 F F F 18 360-379 182 182 182 330 330 330 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 176 176 176 319 319 319 F F F 22 440-459 179 179 179 325 325 325 F F F 23 460-479 179 179 179 325 325 325 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 175 175 175 317 317 317 F F F 26 520-539 180 180 180 327 327 327 F F F 27 540-559 183 183 183 331 331 331 F F F 28 560-579 183 183 183 331 331 331 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 183, Max flush: 331 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.689832156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.734517362Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.735160376Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.735203406Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.735218365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.735237375Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928} {"msg":"Crucible d307eab0-6c08-4d0b-aada-a700974fba7c has session id: 5c2aa18d-76aa-4576-bc3e-9953c452a9eb","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.73585115Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Upstairs opts: Upstairs UUID: d307eab0-6c08-4d0b-aada-a700974fba7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.73587953Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Crucible stats registered with UUID: d307eab0-6c08-4d0b-aada-a700974fba7c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.735894319Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:26.735906849Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736409625Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736434994Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928} {"msg":"d307eab0-6c08-4d0b-aada-a700974fba7c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736514454Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736531544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"0","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736561123Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"0","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736575873Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"1","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736590493Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"1","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736611453Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"2","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736635243Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"2","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736654383Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736684342Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736709872Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34076)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736814341Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:52584)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.736855891Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51193)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.737001129Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.737532955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"1","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.737668633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"0","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.73808267Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"1","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.738112259Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"2","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.738131349Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"0","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.738750884Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.738844023Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.738866152Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"2","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[0]R flush_numbers[0..12]: [328, 331, 331, 328, 321, 326, 330, 318, 326, 326, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739421027Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[0]R generation[0..12]: [181, 183, 183, 181, 176, 180, 182, 176, 180, 180, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739452127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739468567Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[1]R flush_numbers[0..12]: [328, 331, 331, 328, 321, 326, 330, 318, 326, 326, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739483077Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[1]R generation[0..12]: [181, 183, 183, 181, 176, 180, 182, 176, 180, 180, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739497797Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739511177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[2]R flush_numbers[0..12]: [328, 331, 331, 328, 321, 326, 330, 318, 326, 326, 316, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739524457Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[2]R generation[0..12]: [181, 183, 183, 181, 176, 180, 182, 176, 180, 180, 174, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739537806Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739551636Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739564366Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:49:26.739578366Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739597546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739610296Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"":"downstairs","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739623226Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739634965Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"0","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739651075Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"1","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739664195Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"client":"2","":"downstairs","session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739676845Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739688735Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"d307eab0-6c08-4d0b-aada-a700974fba7c is now active with session: 5c2aa18d-76aa-4576-bc3e-9953c452a9eb","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739701205Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739712735Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928,"session_id":"5c2aa18d-76aa-4576-bc3e-9953c452a9eb"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739724505Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928} {"msg":"Activated sub_volume d307eab0-6c08-4d0b-aada-a700974fba7c","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:26.739735305Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 103 len 12288 data: 17 13 17 02/30 Write block 337 len 12288 data: 21 17 16 03/30 Flush 04/30 Write block 291 len 36864 data: 17 18 17 17 14 11 11 10 9 05/30 Write block 118 len 8192 data: 16 17 06/30 Write block 457 len 40960 data: 11 13 11 11 12 13 13 13 13 14 07/30 Flush 08/30 Flush 09/30 Write block 402 len 32768 data: 18 17 14 11 12 9 9 9 10/30 Write block 417 len 40960 data: 12 13 13 14 11 10 9 11 11 15 11/30 Write block 353 len 20480 data: 15 9 11 13 15 12/30 Write block 564 len 8192 data: 12 14 13/30 Read block 156 len 16384 14/30 Read block 562 len 8192 15/30 Read block 359 len 32768 16/30 Write block 500 len 24576 data: 13 14 14 16 13 12 17/30 Read block 14 len 4096 18/30 Write block 527 len 12288 data: 16 14 17 19/30 Read block 496 len 28672 20/30 Read block 143 len 32768 21/30 Write block 406 len 20480 data: 13 10 10 10 10 22/30 Write block 54 len 40960 data: 16 15 16 12 9 7 12 16 17 15 23/30 Read block 558 len 40960 24/30 Write block 202 len 40960 data: 10 12 11 8 11 14 16 17 14 12 25/30 Flush 26/30 Write block 314 len 36864 data: 10 9 12 12 13 16 17 18 16 27/30 Read block 89 len 12288 28/30 Read block 194 len 4096 29/30 Write block 329 len 32768 data: 17 19 19 17 15 17 19 18 30/30 Write block 539 len 36864 data: 15 14 16 14 15 14 15 14 13 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:26.759961162Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1928} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 328 328 328 F F F 1 020-039 183 183 183 331 331 331 F F F 2 040-059 184 184 184 335 335 335 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 184 184 184 333 333 333 F F F 6 120-139 182 182 182 330 330 330 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 180 180 180 326 326 326 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 184 184 184 335 335 335 F F F 11 220-239 179 179 179 325 325 325 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 183 183 183 331 331 331 F F F 14 280-299 184 184 184 333 333 333 F F F 15 300-319 179 179 179 325 325 325 T T T 16 320-339 184 184 184 332 332 332 T T T 17 340-359 184 184 184 335 335 335 F F F 18 360-379 182 182 182 330 330 330 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 184 184 184 335 335 335 F F F 21 420-439 184 184 184 335 335 335 F F F 22 440-459 184 184 184 333 333 333 F F F 23 460-479 184 184 184 333 333 333 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 184 184 184 335 335 335 F F F 26 520-539 184 184 184 335 335 335 T T T 27 540-559 183 183 183 331 331 331 T T T 28 560-579 184 184 184 335 335 335 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 184, Max flush: 335 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 328 328 328 F F F 1 020-039 183 183 183 331 331 331 F F F 2 040-059 184 184 184 335 335 335 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 184 184 184 333 333 333 F F F 6 120-139 182 182 182 330 330 330 F F F 7 140-159 176 176 176 318 318 318 F F F 8 160-179 180 180 180 326 326 326 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 184 184 184 335 335 335 F F F 11 220-239 179 179 179 325 325 325 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 183 183 183 331 331 331 F F F 14 280-299 184 184 184 333 333 333 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 185 185 185 336 336 336 F F F 17 340-359 184 184 184 335 335 335 F F F 18 360-379 182 182 182 330 330 330 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 184 184 184 335 335 335 F F F 21 420-439 184 184 184 335 335 335 F F F 22 440-459 184 184 184 333 333 333 F F F 23 460-479 184 184 184 333 333 333 F F F 24 480-499 174 174 174 316 316 316 F F F 25 500-519 184 184 184 335 335 335 F F F 26 520-539 185 185 185 336 336 336 F F F 27 540-559 185 185 185 336 336 336 F F F 28 560-579 184 184 184 335 335 335 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 185, Max flush: 336 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:30.996864976Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:31.040092245Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.040450511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.071807958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.072358473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.072394002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.072407552Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.072421472Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936} {"msg":"Crucible 26b78dec-a612-4b4c-a3a2-689c01491cad has session id: fd81fbc0-a86a-473e-9577-395738032515","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073044347Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Upstairs opts: Upstairs UUID: 26b78dec-a612-4b4c-a3a2-689c01491cad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073085316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Crucible stats registered with UUID: 26b78dec-a612-4b4c-a3a2-689c01491cad","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073110386Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:31.073131176Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073579452Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073607102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936} {"msg":"26b78dec-a612-4b4c-a3a2-689c01491cad active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073641801Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073657011Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"0","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073687071Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"0","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073701781Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"1","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073716241Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"1","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.07373649Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"2","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.07376389Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"2","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.07378812Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.07381317Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.07383986Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"ds_connection connected from Ok(127.0.0.1:43134)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073868909Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"ds_connection connected from Ok(127.0.0.1:62602)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073902279Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"ds_connection connected from Ok(127.0.0.1:48811)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.073928359Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.074638442Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"2","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.074676382Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"1","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.074820111Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"0","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.075328796Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"1","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.075461585Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"0","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.075490345Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"2","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.075898991Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076067699Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[0]R flush_numbers[0..12]: [328, 331, 335, 335, 321, 333, 330, 318, 326, 326, 335, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076193158Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[0]R generation[0..12]: [181, 183, 184, 184, 176, 184, 182, 176, 180, 180, 184, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076212788Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076227168Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[1]R flush_numbers[0..12]: [328, 331, 335, 335, 321, 333, 330, 318, 326, 326, 335, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076240958Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[1]R generation[0..12]: [181, 183, 184, 184, 176, 184, 182, 176, 180, 180, 184, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076254227Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076268027Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[2]R flush_numbers[0..12]: [328, 331, 335, 335, 321, 333, 330, 318, 326, 326, 335, 325]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076281567Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[2]R generation[0..12]: [181, 183, 184, 184, 176, 184, 182, 176, 180, 180, 184, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076300367Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076314667Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076327297Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076340257Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {Wait for a query_work_queue command to finish before sending IO "msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076358757Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076371786Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"":"downstairs","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076384697Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076396546Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"0","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076409506Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"1","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076422436Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"client":"2","":"downstairs","session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076435306Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076447286Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"26b78dec-a612-4b4c-a3a2-689c01491cad is now active with session: fd81fbc0-a86a-473e-9577-395738032515","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076459876Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076471816Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936,"session_id":"fd81fbc0-a86a-473e-9577-395738032515"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076484016Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936} {"msg":"Activated sub_volume 26b78dec-a612-4b4c-a3a2-689c01491cad","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:31.076494996Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 20480 data: 17 14 13 12 15 02/30 Write block 446 len 24576 data: 16 18 16 14 11 11 03/30 Read block 295 len 12288 04/30 Write block 106 len 4096 data: 21 05/30 Read block 107 len 16384 06/30 Read block 536 len 40960 07/30 Read block 60 len 36864 08/30 Read block 308 len 40960 09/30 Read block 423 len 32768 10/30 Write block 546 len 36864 data: 15 14 13 15 13 18 17 18 16 11/30 Write block 215 len 36864 data: 13 17 16 17 17 20 25 25 24 12/30 Write block 133 len 20480 data: 17 18 18 17 19 13/30 Write block 106 len 20480 data: 22 19 19 17 18 14/30 Read block 423 len 4096 15/30 Write block 23 len 28672 data: 13 12 13 16 18 20 17 16/30 Read block 15 len 32768 17/30 Read block 118 len 28672 18/30 Write block 3 len 12288 data: 9 11 11 19/30 Read block 58 len 28672 20/30 Write block 323 len 28672 data: 15 14 15 11 15 16 18 21/30 Write block 289 len 24576 data: 15 18 18 19 18 18 22/30 Flush 23/30 Flush 24/30 Write block 506 len 40960 data: 13 16 14 15 13 16 16 18 17 15 25/30 Flush 26/30 Write block 140 len 28672 data: 14 14 14 13 14 13 12 27/30 Read block 176 len 4096 28/30 Read block 317 len 40960 29/30 Write block 497 len 40960 data: 13 11 10 14 15 15 18 15 14 14 30/30 Read block 539 len 20480 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:31.098402207Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1936} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 337 337 337 F F F 1 020-039 186 186 186 337 337 337 F F F 2 040-059 184 184 184 335 335 335 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 186 186 186 337 337 337 F F F 6 120-139 186 186 186 337 337 337 F F F 7 140-159 176 176 176 318 318 318 T T T 8 160-179 180 180 180 326 326 326 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 186 186 186 337 337 337 F F F 11 220-239 186 186 186 337 337 337 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 183 183 183 331 331 331 F F F 14 280-299 186 186 186 337 337 337 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 186 186 186 337 337 337 F F F 17 340-359 184 184 184 335 335 335 F F F 18 360-379 182 182 182 330 330 330 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 184 184 184 335 335 335 F F F 21 420-439 184 184 184 335 335 335 F F F 22 440-459 186 186 186 337 337 337 F F F 23 460-479 184 184 184 333 333 333 F F F 24 480-499 174 174 174 316 316 316 T T T 25 500-519 186 186 186 339 339 339 T T T 26 520-539 185 185 185 336 336 336 F F F 27 540-559 186 186 186 337 337 337 F F F 28 560-579 184 184 184 335 335 335 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 186, Max flush: 339 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 337 337 337 F F F 1 020-039 186 186 186 337 337 337 F F F 2 040-059 184 184 184 335 335 335 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 186 186 186 337 337 337 F F F 6 120-139 186 186 186 337 337 337 F F F 7 140-159 187 187 187 340 340 340 F F F 8 160-179 180 180 180 326 326 326 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 186 186 186 337 337 337 F F F 11 220-239 186 186 186 337 337 337 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 183 183 183 331 331 331 F F F 14 280-299 186 186 186 337 337 337 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 186 186 186 337 337 337 F F F 17 340-359 184 184 184 335 335 335 F F F 18 360-379 182 182 182 330 330 330 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 184 184 184 335 335 335 F F F 21 420-439 184 184 184 335 335 335 F F F 22 440-459 186 186 186 337 337 337 F F F 23 460-479 184 184 184 333 333 333 F F F 24 480-499 187 187 187 340 340 340 F F F 25 500-519 187 187 187 340 340 340 F F F 26 520-539 185 185 185 336 336 336 F F F 27 540-559 186 186 186 337 337 337 F F F 28 560-579 184 184 184 335 335 335 F F F 29 580-599 174 174 174 316 316 316 F F F Max gen: 187, Max flush: 340 Begin loop 94 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.355465429Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:35.398783038Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.399223704Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.430257763Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.430906777Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.430944157Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.430962126Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.430976746Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944} {"msg":"Crucible 65c79334-2b6f-4831-80a3-6605da036200 has session id: 660eee2a-1200-4b2c-8cd4-2f2f56d221a3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.431553591Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Upstairs opts: Upstairs UUID: 65c79334-2b6f-4831-80a3-6605da036200, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.431597011Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Crucible stats registered with UUID: 65c79334-2b6f-4831-80a3-6605da036200","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.4316239Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:35.43164585Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432028707Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432063427Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944} {"msg":"65c79334-2b6f-4831-80a3-6605da036200 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432117676Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432145336Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"0","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432177746Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"0","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432194355Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"1","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432218915Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"1","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432246455Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"2","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432272695Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"2","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432307054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432338734Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432368784Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43212)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432393984Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60022)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432419503Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48678)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.432630841Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.433131807Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"1","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.433161297Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"2","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.433420374Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"0","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43385413Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"1","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43388365Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"2","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434188287Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"0","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434489694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434606713Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[0]R flush_numbers[0..12]: [337, 337, 335, 335, 321, 337, 337, 340, 326, 326, 337, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434783322Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[0]R generation[0..12]: [186, 186, 184, 184, 176, 186, 186, 187, 180, 180, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434814912Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434830351Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[1]R flush_numbers[0..12]: [337, 337, 335, 335, 321, 337, 337, 340, 326, 326, 337, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434844491Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[1]R generation[0..12]: [186, 186, 184, 184, 176, 186, 186, 187, 180, 180, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434858251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434871791Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[2]R flush_numbers[0..12]: [337, 337, 335, 335, 321, 337, 337, 340, 326, 326, 337, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434885191Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[2]R generation[0..12]: [186, 186, 184, 184, 176, 186, 186, 187, 180, 180, 186, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434898621Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434911921Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.434924501Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43493729Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43495001Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43497007Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"":"downstairs","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43498277Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43499441Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"0","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43500922Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"1","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43502216Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"client":"2","":"downstairs","session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43503477Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.435046609Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"65c79334-2b6f-4831-80a3-6605da036200 is now active with session: 660eee2a-1200-4b2c-8cd4-2f2f56d221a3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.43505892Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.435070449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944,"session_id":"660eee2a-1200-4b2c-8cd4-2f2f56d221a3"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.435082319Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944} {"msg":"Activated sub_volume 65c79334-2b6f-4831-80a3-6605da036200","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:35.435093119Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 8192 data: 5 7 02/30 Write block 158 len 32768 data: 7 6 5 6 5 5 4 5 03/30 Read block 299 len 8192 04/30 Write block 136 len 32768 data: 18 20 16 14 15 15 15 14 05/30 Read block 10 len 36864 06/30 Read block 328 len 8192 07/30 Read block 436 len 20480 08/30 Read block 152 len 20480 09/30 Read block 304 len 20480 10/30 Read block 159 len 16384 11/30 Flush 12/30 Write block 580 len 32768 data: 18 14 12 14 15 12 12 14 13/30 Write block 401 len 28672 data: 18 19 18 15 12 14 11 14/30 Flush 15/30 Write block 204 len 8192 data: 12 9 16/30 Flush 17/30 Write block 150 len 36864 data: 13 14 14 15 16 16 14 10 8 18/30 Read block 262 len 4096 19/30 Write block 112 len 36864 data: 14 14 12 10 12 14 17 18 22 20/30 Flush 21/30 Write block 155 len 20480 data: 17 15 11 9 7 22/30 Write block 352 len 36864 data: 16 16 10 12 14 16 15 15 15 23/30 Flush 24/30 Flush 25/30 Write block 285 len 24576 data: 12 13 11 13 16 19 26/30 Flush 27/30 Write block 477 len 16384 data: 12 14 9 10 28/30 Write block 368 len 32768 data: 8 11 11 13 17 20 20 20 29/30 Write block 7 len 36864 data: 15 16 15 15 18 20 19 14 18 30/30 Read block 181 len 16384 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 3 2 4 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:35.455515694Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1944} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 341 341 341 T T T 1 020-039 186 186 186 337 337 337 F F F 2 040-059 184 184 184 335 335 335 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 188 188 188 344 344 344 F F F 6 120-139 188 188 188 344 344 344 F F F 7 140-159 188 188 188 345 345 345 F F F 8 160-179 188 188 188 341 341 341 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 188 188 188 343 343 343 F F F 11 220-239 186 186 186 337 337 337 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 183 183 183 331 331 331 F F F 14 280-299 188 188 188 347 347 347 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 186 186 186 337 337 337 F F F 17 340-359 188 188 188 345 345 345 F F F 18 360-379 188 188 188 345 345 345 T T T 19 380-399 182 182 182 330 330 330 F F F 20 400-419 188 188 188 342 342 342 F F F 21 420-439 184 184 184 335 335 335 F F F 22 440-459 186 186 186 337 337 337 F F F 23 460-479 184 184 184 333 333 333 T T T 24 480-499 187 187 187 340 340 340 T T T 25 500-519 187 187 187 340 340 340 F F F 26 520-539 185 185 185 336 336 336 F F F 27 540-559 186 186 186 337 337 337 F F F 28 560-579 184 184 184 335 335 335 F F F 29 580-599 188 188 188 342 342 342 F F F Max gen: 188, Max flush: 347 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 348 348 348 F F F 1 020-039 186 186 186 337 337 337 F F F 2 040-059 184 184 184 335 335 335 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 176 176 176 321 321 321 F F F 5 100-119 188 188 188 344 344 344 F F F 6 120-139 188 188 188 344 344 344 F F F 7 140-159 188 188 188 345 345 345 F F F 8 160-179 188 188 188 341 341 341 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 188 188 188 343 343 343 F F F 11 220-239 186 186 186 337 337 337 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 183 183 183 331 331 331 F F F 14 280-299 188 188 188 347 347 347 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 186 186 186 337 337 337 F F F 17 340-359 188 188 188 345 345 345 F F F 18 360-379 189 189 189 348 348 348 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 188 188 188 342 342 342 F F F 21 420-439 184 184 184 335 335 335 F F F 22 440-459 186 186 186 337 337 337 F F F 23 460-479 189 189 189 348 348 348 F F F 24 480-499 189 189 189 348 348 348 F F F 25 500-519 187 187 187 340 340 340 F F F 26 520-539 185 185 185 336 336 336 F F F 27 540-559 186 186 186 337 337 337 F F F 28 560-579 184 184 184 335 335 335 F F F 29 580-599 188 188 188 342 342 342 F F F Max gen: 189, Max flush: 348 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.729567913Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.776386369Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.777280251Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.77732728Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.77734215Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.77735698Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952} {"msg":"Crucible 3d48663b-8cd7-478e-840d-edcf60c70ac6 has session id: 53c524da-f18e-49ae-830c-abf0f485cea2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778050834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Upstairs opts: Upstairs UUID: 3d48663b-8cd7-478e-840d-edcf60c70ac6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778085343Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Crucible stats registered with UUID: 3d48663b-8cd7-478e-840d-edcf60c70ac6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778100473Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:39.778113183Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778666098Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778703538Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952} {"msg":"3d48663b-8cd7-478e-840d-edcf60c70ac6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778773957Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778793177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"0","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778828087Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"0","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778843597Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"1","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778858966Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"1","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778872366Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"2","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778886836Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"2","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778905156Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778930046Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778955376Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"ds_connection connected from Ok(127.0.0.1:46433)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.778982135Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41168)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.779008955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"ds_connection connected from Ok(127.0.0.1:40119)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.779174363Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.779787248Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"0","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.779815018Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"1","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.779830628Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"2","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.780515441Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"1","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.780545751Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"2","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.780572931Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"0","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781131746Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781244515Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[0]R flush_numbers[0..12]: [348, 337, 335, 335, 321, 344, 344, 345, 341, 326, 343, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781316214Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[0]R generation[0..12]: [189, 186, 184, 184, 176, 188, 188, 188, 188, 180, 188, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781336144Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781350544Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[1]R flush_numbers[0..12]: [348, 337, 335, 335, 321, 344, 344, 345, 341, 326, 343, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781364384Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[1]R generation[0..12]: [189, 186, 184, 184, 176, 188, 188, 188, 188, 180, 188, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781378043Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781391703Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[2]R flush_numbers[0..12]: [348, 337, 335, 335, 321, 344, 344, 345, 341, 326, 343, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781404873Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[2]R generation[0..12]: [189, 186, 184, 184, 176, 188, 188, 188, 188, 180, 188, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781418103Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781431703Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781447423Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781474243Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781487562Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781500173Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"":"downstairs","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781512842Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781524482Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"0","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781537262Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"1","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781557902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"client":"2","":"downstairs","session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781570892Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781582732Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"3d48663b-8cd7-478e-840d-edcf60c70ac6 is now active with session: 53c524da-f18e-49ae-830c-abf0f485cea2","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781595201Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781606641Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952,"session_id":"53c524da-f18e-49ae-830c-abf0f485cea2"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781618601Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952} {"msg":"Activated sub_volume 3d48663b-8cd7-478e-840d-edcf60c70ac6","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:39.781629511Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 261 len 24576 data: 10 14 14 20 20 16 02/30 Write block 520 len 4096 data: 14 03/30 Read block 547 len 24576 04/30 Read block 371 len 8192 05/30 Read block 78 len 4096 06/30 Write block 544 len 4096 data: 15 07/30 Write block 84 len 4096 data: 16 08/30 Read block 317 len 40960 09/30 Flush 10/30 Write block 359 len 32768 data: 16 16 11 12 12 11 8 10 11/30 Write block 577 len 16384 data: 18 18 21 19 12/30 Write block 584 len 36864 data: 16 13 13 15 12 10 11 12 16 13/30 Read block 1 len 36864 14/30 Read block 477 len 28672 15/30 Write block 573 len 28672 data: 12 14 16 15 19 19 22 16/30 Write block 414 len 8192 data: 11 13 17/30 Write block 532 len 32768 data: 14 14 15 18 15 13 14 16 18/30 Read block 178 len 12288 19/30 Flush 20/30 Write block 502 len 12288 data: 16 19 16 21/30 Write block 563 len 20480 data: 12 13 15 11 15 22/30 Read block 112 len 40960 23/30 Read block 517 len 8192 24/30 Write block 136 len 16384 data: 19 21 17 15 25/30 Read block 514 len 20480 26/30 Write block 438 len 16384 data: 12 13 14 12 27/30 Write block 55 len 16384 data: 16 17 13 10 28/30 Read block 374 len 4096 29/30 Write block 324 len 32768 data: 15 16 12 16 17 19 20 20 30/30 Read block 75 len 8192 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 3 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Done Sent false 1029 Acked Read 2 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 9 11 9 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:39.800920357Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1952} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 348 348 348 F F F 1 020-039 186 186 186 337 337 337 F F F 2 040-059 184 184 184 335 335 335 T T T 3 060-079 184 184 184 335 335 335 F F F 4 080-099 190 190 190 349 349 349 F F F 5 100-119 188 188 188 344 344 344 F F F 6 120-139 188 188 188 344 344 344 T T T 7 140-159 188 188 188 345 345 345 F F F 8 160-179 188 188 188 341 341 341 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 188 188 188 343 343 343 F F F 11 220-239 186 186 186 337 337 337 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 190 190 190 349 349 349 F F F 14 280-299 188 188 188 347 347 347 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 186 186 186 337 337 337 T T T 17 340-359 190 190 190 350 350 350 F F F 18 360-379 190 190 190 350 350 350 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 190 190 190 350 350 350 F F F 21 420-439 184 184 184 335 335 335 T T T 22 440-459 186 186 186 337 337 337 T T T 23 460-479 189 189 189 348 348 348 F F F 24 480-499 189 189 189 348 348 348 F F F 25 500-519 187 187 187 340 340 340 T T T 26 520-539 190 190 190 350 350 350 F F F 27 540-559 190 190 190 349 349 349 F F F 28 560-579 190 190 190 350 350 350 T T T 29 580-599 190 190 190 350 350 350 F F F Max gen: 190, Max flush: 350 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 348 348 348 F F F 1 020-039 186 186 186 337 337 337 F F F 2 040-059 191 191 191 351 351 351 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 190 190 190 349 349 349 F F F 5 100-119 188 188 188 344 344 344 F F F 6 120-139 191 191 191 351 351 351 F F F 7 140-159 188 188 188 345 345 345 F F F 8 160-179 188 188 188 341 341 341 F F F 9 180-199 180 180 180 326 326 326 F F F 10 200-219 188 188 188 343 343 343 F F F 11 220-239 186 186 186 337 337 337 F F F 12 240-259 180 180 180 327 327 327 F F F 13 260-279 190 190 190 349 349 349 F F F 14 280-299 188 188 188 347 347 347 F F F 15 300-319 185 185 185 336 336 336 F F F 16 320-339 191 191 191 351 351 351 F F F 17 340-359 190 190 190 350 350 350 F F F 18 360-379 190 190 190 350 350 350 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 190 190 190 350 350 350 F F F 21 420-439 191 191 191 351 351 351 F F F 22 440-459 191 191 191 351 351 351 F F F 23 460-479 189 189 189 348 348 348 F F F 24 480-499 189 189 189 348 348 348 F F F 25 500-519 191 191 191 351 351 351 F F F 26 520-539 190 190 190 350 350 350 F F F 27 540-559 190 190 190 349 349 349 F F F 28 560-579 191 191 191 351 351 351 F F F 29 580-599 190 190 190 350 350 350 F F F Max gen: 191, Max flush: 351 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.059356477Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.105101973Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.105936655Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.105988885Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.106007565Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.106025574Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960} {"msg":"Crucible c0c91995-3ea0-4f3b-aa6a-0b2a348e1753 has session id: 058eea5f-2563-4543-a32d-ef0a4423ff07","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.106583769Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Upstairs opts: Upstairs UUID: c0c91995-3ea0-4f3b-aa6a-0b2a348e1753, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.106630349Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Crucible stats registered with UUID: c0c91995-3ea0-4f3b-aa6a-0b2a348e1753","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.106654939Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:44.106675428Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107286733Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107321453Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960} {"msg":"c0c91995-3ea0-4f3b-aa6a-0b2a348e1753 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107375962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107402712Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"0","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107433892Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"0","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107448092Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"1","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107462281Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"1","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107478611Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"2","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107503331Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"2","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107527151Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107550671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.10757501Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"ds_connection connected from Ok(127.0.0.1:34129)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107627Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"ds_connection connected from Ok(127.0.0.1:32807)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.10765832Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"ds_connection connected from Ok(127.0.0.1:39678)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.107827088Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.108445572Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"2","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.108473772Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"0","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.108609451Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"1","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.109121486Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"2","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.109279305Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"1","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.109317025Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"0","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.109656901Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.10983618Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[0]R flush_numbers[0..12]: [348, 337, 351, 335, 349, 344, 351, 345, 341, 326, 343, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.109982759Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[0]R generation[0..12]: [189, 186, 191, 184, 190, 188, 191, 188, 188, 180, 188, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110012098Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110028238Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[1]R flush_numbers[0..12]: [348, 337, 351, 335, 349, 344, 351, 345, 341, 326, 343, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110047138Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[1]R generation[0..12]: [189, 186, 191, 184, 190, 188, 191, 188, 188, 180, 188, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110061218Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110075318Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[2]R flush_numbers[0..12]: [348, 337, 351, 335, 349, 344, 351, 345, 341, 326, 343, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110089948Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[2]R generation[0..12]: [189, 186, 191, 184, 190, 188, 191, 188, 188, 180, 188, 186]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110103837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110117647Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110130057Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110143127Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110155727Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"Wait for a query_work_queue command to finish before sending IO } {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110175857Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"":"downstairs","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110188657Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110200197Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"0","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110213096Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"1","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110225787Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"client":"2","":"downstairs","session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110238456Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110253316Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"c0c91995-3ea0-4f3b-aa6a-0b2a348e1753 is now active with session: 058eea5f-2563-4543-a32d-ef0a4423ff07","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110266116Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110277726Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960,"session_id":"058eea5f-2563-4543-a32d-ef0a4423ff07"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110289656Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960} {"msg":"Activated sub_volume c0c91995-3ea0-4f3b-aa6a-0b2a348e1753","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:44.110300586Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 4096 data: 7 02/30 Write block 206 len 12288 data: 12 15 17 03/30 Flush 04/30 Write block 475 len 28672 data: 9 10 13 15 10 11 10 05/30 Read block 300 len 28672 06/30 Write block 291 len 28672 data: 19 20 19 19 15 12 12 07/30 Write block 345 len 36864 data: 17 14 14 15 14 19 18 17 17 08/30 Write block 481 len 8192 data: 11 7 09/30 Read block 449 len 36864 10/30 Write block 458 len 36864 data: 14 12 12 13 14 14 14 14 15 11/30 Write block 247 len 32768 data: 10 13 15 13 18 19 20 18 12/30 Write block 374 len 16384 data: 21 21 16 18 13/30 Write block 310 len 40960 data: 14 13 13 13 11 10 13 13 14 17 14/30 Write block 423 len 24576 data: 10 12 12 16 10 11 15/30 Flush 16/30 Write block 494 len 36864 data: 15 14 11 14 12 11 15 16 17 17/30 Write block 172 len 36864 data: 9 11 11 13 12 11 13 14 8 18/30 Write block 24 len 12288 data: 13 14 17 19/30 Read block 500 len 40960 20/30 Write block 585 len 20480 data: 14 14 16 13 11 21/30 Read block 317 len 12288 22/30 Read block 466 len 20480 23/30 Read block 366 len 16384 24/30 Read block 561 len 40960 25/30 Write block 232 len 8192 data: 9 12 26/30 Write block 197 len 20480 data: 8 10 11 12 9 27/30 Read block 398 len 28672 28/30 Write block 141 len 20480 data: 16 16 15 15 14 29/30 Write block 353 len 28672 data: 18 11 13 15 17 16 17 30/30 Write block 234 len 8192 data: 14 17 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:44.129340274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1960} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 352 352 352 F F F 1 020-039 186 186 186 337 337 337 T T T 2 040-059 191 191 191 351 351 351 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 190 190 190 349 349 349 F F F 5 100-119 188 188 188 344 344 344 F F F 6 120-139 191 191 191 351 351 351 F F F 7 140-159 188 188 188 345 345 345 T T T 8 160-179 188 188 188 341 341 341 T T T 9 180-199 180 180 180 326 326 326 T T T 10 200-219 192 192 192 352 352 352 T T T 11 220-239 186 186 186 337 337 337 T T T 12 240-259 192 192 192 353 353 353 F F F 13 260-279 190 190 190 349 349 349 F F F 14 280-299 192 192 192 353 353 353 F F F 15 300-319 192 192 192 353 353 353 F F F 16 320-339 191 191 191 351 351 351 F F F 17 340-359 192 192 192 353 353 353 T T T 18 360-379 192 192 192 353 353 353 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 190 190 190 350 350 350 F F F 21 420-439 192 192 192 353 353 353 F F F 22 440-459 192 192 192 353 353 353 F F F 23 460-479 192 192 192 353 353 353 F F F 24 480-499 192 192 192 353 353 353 T T T 25 500-519 191 191 191 351 351 351 T T T 26 520-539 190 190 190 350 350 350 F F F 27 540-559 190 190 190 349 349 349 F F F 28 560-579 191 191 191 351 351 351 F F F 29 580-599 190 190 190 350 350 350 T T T Max gen: 192, Max flush: 353 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 352 352 352 F F F 1 020-039 193 193 193 354 354 354 F F F 2 040-059 191 191 191 351 351 351 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 190 190 190 349 349 349 F F F 5 100-119 188 188 188 344 344 344 F F F 6 120-139 191 191 191 351 351 351 F F F 7 140-159 193 193 193 354 354 354 F F F 8 160-179 193 193 193 354 354 354 F F F 9 180-199 193 193 193 354 354 354 F F F 10 200-219 193 193 193 354 354 354 F F F 11 220-239 193 193 193 354 354 354 F F F 12 240-259 192 192 192 353 353 353 F F F 13 260-279 190 190 190 349 349 349 F F F 14 280-299 192 192 192 353 353 353 F F F 15 300-319 192 192 192 353 353 353 F F F 16 320-339 191 191 191 351 351 351 F F F 17 340-359 193 193 193 354 354 354 F F F 18 360-379 192 192 192 353 353 353 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 190 190 190 350 350 350 F F F 21 420-439 192 192 192 353 353 353 F F F 22 440-459 192 192 192 353 353 353 F F F 23 460-479 192 192 192 353 353 353 F F F 24 480-499 193 193 193 354 354 354 F F F 25 500-519 193 193 193 354 354 354 F F F 26 520-539 190 190 190 350 350 350 F F F 27 540-559 190 190 190 349 349 349 F F F 28 560-579 191 191 191 351 351 351 F F F 29 580-599 193 193 193 354 354 354 F F F Max gen: 193, Max flush: 354 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.402217383Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.446793619Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.447376794Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.447428424Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.447446793Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.447464813Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968} {"msg":"Crucible a52b6a30-75dc-4296-b534-ad74718017c7 has session id: 87d1616c-3d9c-4683-bb14-d93fdca99052","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.447988379Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Upstairs opts: Upstairs UUID: a52b6a30-75dc-4296-b534-ad74718017c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448028248Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Crucible stats registered with UUID: a52b6a30-75dc-4296-b534-ad74718017c7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448047348Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:48.448059968Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448495504Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448519584Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968} {"msg":"a52b6a30-75dc-4296-b534-ad74718017c7 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448581273Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448598853Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"0","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448631303Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"0","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448653633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"1","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448680572Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"1","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448757961Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"2","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448794681Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"2","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448820821Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448846861Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.448871171Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"ds_connection connected from Ok(127.0.0.1:52924)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.44889447Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"ds_connection connected from Ok(127.0.0.1:51527)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.449059579Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"ds_connection connected from Ok(127.0.0.1:33359)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.449102598Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.449618054Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"2","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.449646984Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"0","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.45003906Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"1","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.450279048Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"0","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.450311718Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"2","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.450673474Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"1","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.450914962Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.450945352Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[0]R flush_numbers[0..12]: [352, 354, 351, 335, 349, 344, 351, 354, 354, 354, 354, 354]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451232979Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[0]R generation[0..12]: [192, 193, 191, 184, 190, 188, 191, 193, 193, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451257759Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451272879Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[1]R flush_numbers[0..12]: [352, 354, 351, 335, 349, 344, 351, 354, 354, 354, 354, 354]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451292899Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[1]R generation[0..12]: [192, 193, 191, 184, 190, 188, 191, 193, 193, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451318628Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451346538Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[2]R flush_numbers[0..12]: [352, 354, 351, 335, 349, 344, 351, 354, 354, 354, 354, 354]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451371978Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[2]R generation[0..12]: [192, 193, 191, 184, 190, 188, 191, 193, 193, 193, 193, 193]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T01:49:48.451397378Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451432867Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451456847Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451471337Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Next flush: 355","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451484307Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451506937Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"":"downstairs","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451531696Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451554236Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"0","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451570076Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"1","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451583086Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"client":"2","":"downstairs","session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451604936Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451627646Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"a52b6a30-75dc-4296-b534-ad74718017c7 is now active with session: 87d1616c-3d9c-4683-bb14-d93fdca99052","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451651775Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451670545Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968,"session_id":"87d1616c-3d9c-4683-bb14-d93fdca99052"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451683315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968} {"msg":"Activated sub_volume a52b6a30-75dc-4296-b534-ad74718017c7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:48.451697415Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 520 len 8192 data: 15 17 02/30 Read block 514 len 16384 03/30 Write block 516 len 36864 data: 18 18 14 15 16 18 16 13 16 04/30 Write block 344 len 28672 data: 14 18 15 15 16 15 20 05/30 Write block 512 len 4096 data: 17 06/30 Read block 570 len 12288 07/30 Write block 464 len 8192 data: 15 15 08/30 Write block 10 len 4096 data: 16 09/30 Read block 285 len 16384 10/30 Read block 445 len 32768 11/30 Read block 240 len 24576 12/30 Read block 181 len 32768 13/30 Read block 126 len 8192 14/30 Read block 434 len 32768 15/30 Write block 231 len 20480 data: 12 10 13 15 18 16/30 Read block 474 len 32768 17/30 Write block 242 len 16384 data: 16 12 12 12 18/30 Read block 176 len 36864 19/30 Read block 452 len 20480 20/30 Read block 53 len 20480 21/30 Read block 229 len 24576 22/30 Write block 271 len 28672 data: 17 16 18 17 17 17 18 23/30 Read block 72 len 28672 24/30 Write block 26 len 12288 data: 18 19 21 25/30 Flush 26/30 Write block 504 len 36864 data: 17 15 15 17 15 16 14 17 18 27/30 Write block 95 len 20480 data: 13 12 13 16 16 28/30 Write block 140 len 36864 data: 16 17 17 16 16 15 13 12 13 29/30 Write block 134 len 12288 data: 19 19 20 30/30 Write block 361 len 28672 data: 12 13 13 12 9 11 8 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:48.470765102Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1968} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 355 355 355 F F F 1 020-039 194 194 194 355 355 355 F F F 2 040-059 191 191 191 351 351 351 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 190 190 190 349 349 349 T T T 5 100-119 188 188 188 344 344 344 F F F 6 120-139 191 191 191 351 351 351 T T T 7 140-159 193 193 193 354 354 354 T T T 8 160-179 193 193 193 354 354 354 F F F 9 180-199 193 193 193 354 354 354 F F F 10 200-219 193 193 193 354 354 354 F F F 11 220-239 194 194 194 355 355 355 F F F 12 240-259 194 194 194 355 355 355 F F F 13 260-279 194 194 194 355 355 355 F F F 14 280-299 192 192 192 353 353 353 F F F 15 300-319 192 192 192 353 353 353 F F F 16 320-339 191 191 191 351 351 351 F F F 17 340-359 194 194 194 355 355 355 F F F 18 360-379 192 192 192 353 353 353 T T T 19 380-399 182 182 182 330 330 330 F F F 20 400-419 190 190 190 350 350 350 F F F 21 420-439 192 192 192 353 353 353 F F F 22 440-459 192 192 192 353 353 353 F F F 23 460-479 194 194 194 355 355 355 F F F 24 480-499 193 193 193 354 354 354 F F F 25 500-519 194 194 194 355 355 355 T T T 26 520-539 194 194 194 355 355 355 F F F 27 540-559 190 190 190 349 349 349 F F F 28 560-579 191 191 191 351 351 351 F F F 29 580-599 193 193 193 354 354 354 F F F Max gen: 194, Max flush: 355 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 355 355 355 F F F 1 020-039 194 194 194 355 355 355 F F F 2 040-059 191 191 191 351 351 351 F F F 3 060-079 184 184 184 335 335 335 F F F 4 080-099 195 195 195 356 356 356 F F F 5 100-119 188 188 188 344 344 344 F F F 6 120-139 195 195 195 356 356 356 F F F 7 140-159 195 195 195 356 356 356 F F F 8 160-179 193 193 193 354 354 354 F F F 9 180-199 193 193 193 354 354 354 F F F 10 200-219 193 193 193 354 354 354 F F F 11 220-239 194 194 194 355 355 355 F F F 12 240-259 194 194 194 355 355 355 F F F 13 260-279 194 194 194 355 355 355 F F F 14 280-299 192 192 192 353 353 353 F F F 15 300-319 192 192 192 353 353 353 F F F 16 320-339 191 191 191 351 351 351 F F F 17 340-359 194 194 194 355 355 355 F F F 18 360-379 195 195 195 356 356 356 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 190 190 190 350 350 350 F F F 21 420-439 192 192 192 353 353 353 F F F 22 440-459 192 192 192 353 353 353 F F F 23 460-479 194 194 194 355 355 355 F F F 24 480-499 193 193 193 354 354 354 F F F 25 500-519 195 195 195 356 356 356 F F F 26 520-539 194 194 194 355 355 355 F F F 27 540-559 190 190 190 349 349 349 F F F 28 560-579 191 191 191 351 351 351 F F F 29 580-599 193 193 193 354 354 354 F F F Max gen: 195, Max flush: 356 Begin loop 98 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.710714479Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.757928772Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.758653596Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.758699145Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.758715385Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.758734935Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976} {"msg":"Crucible 0eac3b15-5a18-43c2-9a42-a9452a67f873 has session id: 714418f7-82b3-452b-b35c-a5fc1bcdf461","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.75926018Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Upstairs opts: Upstairs UUID: 0eac3b15-5a18-43c2-9a42-a9452a67f873, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.75929614Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Crucible stats registered with UUID: 0eac3b15-5a18-43c2-9a42-a9452a67f873","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.75931605Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:52.759332299Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759767315Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759794405Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976} {"msg":"0eac3b15-5a18-43c2-9a42-a9452a67f873 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759851955Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759871565Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"0","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759901334Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"0","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759915294Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"1","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759929274Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"1","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759942104Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"2","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759955624Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"2","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759968244Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759981254Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.759994513Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"ds_connection connected from Ok(127.0.0.1:60870)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.760115672Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"ds_connection connected from Ok(127.0.0.1:42486)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.760157562Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"ds_connection connected from Ok(127.0.0.1:46580)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.760183632Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.760930065Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"2","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.760970485Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"1","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.761000734Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"0","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.761570529Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"2","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.761606719Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"0","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.761727848Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"1","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762241423Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762264533Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[0]R flush_numbers[0..12]: [355, 355, 351, 335, 356, 344, 356, 356, 354, 354, 354, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762325523Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[0]R generation[0..12]: [194, 194, 191, 184, 195, 188, 195, 195, 193, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762345002Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762359572Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[1]R flush_numbers[0..12]: [355, 355, 351, 335, 356, 344, 356, 356, 354, 354, 354, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762373322Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[1]R generation[0..12]: [194, 194, 191, 184, 195, 188, 195, 195, 193, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762390322Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762404362Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[2]R flush_numbers[0..12]: [355, 355, 351, 335, 356, 344, 356, 356, 354, 354, 354, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762417731Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[2]R generation[0..12]: [194, 194, 191, 184, 195, 188, 195, 195, 193, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762431122Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762444921Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762457361Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762470311Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Next flush: 357","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762483061Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.762495671Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"":"downstairs","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2025-03-18T01:49:52.762509391Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76252795Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"0","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76254237Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"1","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76255523Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"client":"2","":"downstairs","session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76256801Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76258177Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"0eac3b15-5a18-43c2-9a42-a9452a67f873 is now active with session: 714418f7-82b3-452b-b35c-a5fc1bcdf461","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76259445Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76260608Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976,"session_id":"714418f7-82b3-452b-b35c-a5fc1bcdf461"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76261814Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976} {"msg":"Activated sub_volume 0eac3b15-5a18-43c2-9a42-a9452a67f873","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:52.76262902Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 86 len 16384 data: 16 14 16 20 02/30 Read block 217 len 8192 03/30 Flush 04/30 Write block 550 len 12288 data: 14 19 18 05/30 Write block 319 len 4096 data: 18 06/30 Write block 68 len 32768 data: 19 19 15 12 12 14 13 15 07/30 Read block 473 len 16384 08/30 Write block 296 len 4096 data: 13 09/30 Write block 112 len 4096 data: 15 10/30 Read block 475 len 4096 11/30 Read block 25 len 28672 12/30 Write block 353 len 8192 data: 19 12 13/30 Write block 401 len 4096 data: 19 14/30 Read block 571 len 16384 15/30 Flush 16/30 Write block 25 len 32768 data: 15 19 20 22 18 17 15 14 17/30 Flush 18/30 Write block 255 len 36864 data: 18 16 14 13 10 12 11 15 15 19/30 Read block 82 len 4096 20/30 Write block 472 len 20480 data: 11 9 8 10 11 21/30 Read block 573 len 32768 22/30 Write block 500 len 28672 data: 16 17 18 20 18 16 16 23/30 Read block 260 len 16384 24/30 Flush 25/30 Flush 26/30 Write block 275 len 16384 data: 18 18 19 19 27/30 Write block 529 len 28672 data: 18 16 13 15 15 16 19 28/30 Read block 64 len 28672 29/30 Write block 58 len 40960 data: 11 8 13 17 18 16 17 20 22 19 30/30 Read block 523 len 12288 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:52.781581858Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1976} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 355 355 355 F F F 1 020-039 196 196 196 359 359 359 F F F 2 040-059 191 191 191 351 351 351 T T T 3 060-079 196 196 196 358 358 358 T T T 4 080-099 196 196 196 357 357 357 F F F 5 100-119 196 196 196 358 358 358 F F F 6 120-139 195 195 195 356 356 356 F F F 7 140-159 195 195 195 356 356 356 F F F 8 160-179 193 193 193 354 354 354 F F F 9 180-199 193 193 193 354 354 354 F F F 10 200-219 193 193 193 354 354 354 F F F 11 220-239 194 194 194 355 355 355 F F F 12 240-259 196 196 196 360 360 360 F F F 13 260-279 196 196 196 360 360 360 T T T 14 280-299 196 196 196 358 358 358 F F F 15 300-319 196 196 196 358 358 358 F F F 16 320-339 191 191 191 351 351 351 F F F 17 340-359 196 196 196 358 358 358 F F F 18 360-379 195 195 195 356 356 356 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 196 196 196 358 358 358 F F F 21 420-439 192 192 192 353 353 353 F F F 22 440-459 192 192 192 353 353 353 F F F 23 460-479 196 196 196 360 360 360 F F F 24 480-499 193 193 193 354 354 354 F F F 25 500-519 196 196 196 360 360 360 F F F 26 520-539 194 194 194 355 355 355 T T T 27 540-559 196 196 196 358 358 358 F F F 28 560-579 191 191 191 351 351 351 F F F 29 580-599 193 193 193 354 354 354 F F F Max gen: 196, Max flush: 360 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 355 355 355 F F F 1 020-039 196 196 196 359 359 359 F F F 2 040-059 197 197 197 361 361 361 F F F 3 060-079 197 197 197 361 361 361 F F F 4 080-099 196 196 196 357 357 357 F F F 5 100-119 196 196 196 358 358 358 F F F 6 120-139 195 195 195 356 356 356 F F F 7 140-159 195 195 195 356 356 356 F F F 8 160-179 193 193 193 354 354 354 F F F 9 180-199 193 193 193 354 354 354 F F F 10 200-219 193 193 193 354 354 354 F F F 11 220-239 194 194 194 355 355 355 F F F 12 240-259 196 196 196 360 360 360 F F F 13 260-279 197 197 197 361 361 361 F F F 14 280-299 196 196 196 358 358 358 F F F 15 300-319 196 196 196 358 358 358 F F F 16 320-339 191 191 191 351 351 351 F F F 17 340-359 196 196 196 358 358 358 F F F 18 360-379 195 195 195 356 356 356 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 196 196 196 358 358 358 F F F 21 420-439 192 192 192 353 353 353 F F F 22 440-459 192 192 192 353 353 353 F F F 23 460-479 196 196 196 360 360 360 F F F 24 480-499 193 193 193 354 354 354 F F F 25 500-519 196 196 196 360 360 360 F F F 26 520-539 197 197 197 361 361 361 F F F 27 540-559 196 196 196 358 358 358 F F F 28 560-579 191 191 191 351 351 351 F F F 29 580-599 193 193 193 354 354 354 F F F Max gen: 197, Max flush: 361 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.017872398Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:57.06073256Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.061087837Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.092353834Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.092919449Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d0c577cf78d28348432c142546caf529999ea42a\",\n git_commit_timestamp: \"2025-03-18T01:24:26.000000000Z\",\n git_branch: \"renovate/http-1.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.092954568Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.092968168Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.092982318Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984} {"msg":"Crucible 303470ed-9273-4d46-860a-935457adac2d has session id: c03dd825-d2ed-4bd8-baa4-b3b0996013f7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.093580353Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Upstairs opts: Upstairs UUID: 303470ed-9273-4d46-860a-935457adac2d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.093611703Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Crucible stats registered with UUID: 303470ed-9273-4d46-860a-935457adac2d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.093627882Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:57.093641752Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094000509Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094025619Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984} {"msg":"303470ed-9273-4d46-860a-935457adac2d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094085398Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094104338Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"0","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094133768Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"0","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094151928Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"1","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094166108Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"1","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094179097Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"2","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094193917Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"2","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094206737Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094219967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094242967Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60452)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094268927Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57749)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094296116Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53572)","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.094473005Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.095127419Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"2","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.095156139Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"1","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.095171679Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"0","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.095768763Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"1","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.095905812Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"0","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.095950712Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"2","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096329008Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096528066Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"[0]R flush_numbers[0..12]: [355, 359, 361, 361, 357, 358, 356, 356, 354, 354, 354, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096665015Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"[0]R generation[0..12]: [194, 196, 197, 197, 196, 196, 195, 195, 193, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096686105Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096700665Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"[1]R flush_numbers[0..12]: [355, 359, 361, 361, 357, 358, 356, 356, 354, 354, 354, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096716334Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"[1]R generation[0..12]: [194, 196, 197, 197, 196, 196, 195, 195, 193, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096740704Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096768404Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"[2]R flush_numbers[0..12]: [355, 359, 361, 361, 357, 358, 356, 356, 354, 354, 354, 355]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096795204Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"[2]R generation[0..12]: [194, 196, 197, 197, 196, 196, 195, 195, 193, 193, 193, 194]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096816614Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096847723Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096873633Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096894543Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Next flush: 362","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096909263Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096927523Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"":"downstairs","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096951052Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096973572Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"0","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.096995492Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"1","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.097009172Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"client":"2","":"downstairs","session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.097028812Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.097050802Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"303470ed-9273-4d46-860a-935457adac2d is now active with session: c03dd825-d2ed-4bd8-baa4-b3b0996013f7","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.097073461Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.097095721Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984,"session_id":"c03dd825-d2ed-4bd8-baa4-b3b0996013f7"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.097109021Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984} {"msg":"Activated sub_volume 303470ed-9273-4d46-860a-935457adac2d","v":0,"name":"crucible","level":30,"time":"2025-03-18T01:49:57.097119941Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 178 len 20480 data: 14 15 9 10 10 02/30 Flush 03/30 Write block 547 len 36864 data: 15 14 16 15 20 19 19 17 16 04/30 Write block 98 len 20480 data: 17 17 18 18 15 05/30 Read block 266 len 28672 06/30 Write block 398 len 16384 data: 18 18 17 20 07/30 Read block 334 len 8192 08/30 Read block 327 len 4096 09/30 Write block 332 len 20480 data: 18 16 18 20 19 10/30 Read block 498 len 16384 11/30 Read block 140 len 28672 12/30 Read block 101 len 40960 13/30 Write block 287 len 28672 data: 12 14 17 20 20 21 20 14/30 Read block 342 len 4096 15/30 Write block 7 len 36864 data: 16 17 16 17 19 21 20 15 19 16/30 Write block 46 len 28672 data: 13 12 12 12 12 15 17 17/30 Read block 438 len 12288 18/30 Read block 23 len 36864 19/30 Write block 281 len 12288 data: 20 20 18 20/30 Read block 529 len 8192 21/30 Write block 444 len 12288 data: 17 16 17 22/30 Read block 333 len 36864 23/30 Read block 21 len 12288 24/30 Read block 111 len 40960 25/30 Read block 459 len 16384 26/30 Read block 180 len 20480 27/30 Read block 391 len 4096 28/30 Write block 385 len 36864 data: 15 14 19 22 23 25 21 21 20 29/30 Read block 457 len 16384 30/30 Read block 567 len 20480 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 9 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 7 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 28 27 27 82 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T01:49:57.118313039Z","hostname":"w-01JPKEFKCDJDSKHT6J57X16Y8T","pid":1984} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 194 194 194 355 355 355 T T T 1 020-039 196 196 196 359 359 359 F F F 2 040-059 197 197 197 361 361 361 T T T 3 060-079 197 197 197 361 361 361 F F F 4 080-099 196 196 196 357 357 357 T T T 5 100-119 196 196 196 358 358 358 T T T 6 120-139 195 195 195 356 356 356 F F F 7 140-159 195 195 195 356 356 356 F F F 8 160-179 198 198 198 362 362 362 F F F 9 180-199 198 198 198 362 362 362 F F F 10 200-219 193 193 193 354 354 354 F F F 11 220-239 194 194 194 355 355 355 F F F 12 240-259 196 196 196 360 360 360 F F F 13 260-279 197 197 197 361 361 361 F F F 14 280-299 196 196 196 358 358 358 T T T 15 300-319 196 196 196 358 358 358 F F F 16 320-339 191 191 191 351 351 351 T T T 17 340-359 196 196 196 358 358 358 F F F 18 360-379 195 195 195 356 356 356 F F F 19 380-399 182 182 182 330 330 330 T T T 20 400-419 196 196 196 358 358 358 T T T 21 420-439 192 192 192 353 353 353 F F F 22 440-459 192 192 192 353 353 353 T T T 23 460-479 196 196 196 360 360 360 F F F 24 480-499 193 193 193 354 354 354 F F F 25 500-519 196 196 196 360 360 360 F F F 26 520-539 197 197 197 361 361 361 F F F 27 540-559 196 196 196 358 358 358 T T T 28 560-579 191 191 191 351 351 351 F F F 29 580-599 193 193 193 354 354 354 F F F Max gen: 198, Max flush: 362 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 199 199 199 363 363 363 F F F 1 020-039 196 196 196 359 359 359 F F F 2 040-059 199 199 199 363 363 363 F F F 3 060-079 197 197 197 361 361 361 F F F 4 080-099 199 199 199 363 363 363 F F F 5 100-119 199 199 199 363 363 363 F F F 6 120-139 195 195 195 356 356 356 F F F 7 140-159 195 195 195 356 356 356 F F F 8 160-179 198 198 198 362 362 362 F F F 9 180-199 198 198 198 362 362 362 F F F 10 200-219 193 193 193 354 354 354 F F F 11 220-239 194 194 194 355 355 355 F F F 12 240-259 196 196 196 360 360 360 F F F 13 260-279 197 197 197 361 361 361 F F F 14 280-299 199 199 199 363 363 363 F F F 15 300-319 196 196 196 358 358 358 F F F 16 320-339 199 199 199 363 363 363 F F F 17 340-359 196 196 196 358 358 358 F F F 18 360-379 195 195 195 356 356 356 F F F 19 380-399 199 199 199 363 363 363 F F F 20 400-419 199 199 199 363 363 363 F F F 21 420-439 192 192 192 353 353 353 F F F 22 440-459 199 199 199 363 363 363 F F F 23 460-479 196 196 196 360 360 360 F F F 24 480-499 193 193 193 354 354 354 F F F 25 500-519 196 196 196 360 360 360 F F F 26 520-539 197 197 197 361 361 361 F F F 27 540-559 199 199 199 363 363 363 F F F 28 560-579 191 191 191 351 351 351 F F F 29 580-599 193 193 193 354 354 354 F F F Max gen: 199, Max flush: 363 9:04 Test duration Test completed real 9:04.130918173 user 48.737950597 sys 19.038652365 trap 0.064379653 tflt 0.096482305 dflt 0.125889000 kflt 0.006037187 lock 14:29:08.744216706 slp 44:20.969599291 lat 5.539828624 stop 0.060872790