Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.27402684 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.020510895 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.020252991 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1193 1194 1195 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.785248408Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.841135977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.843171243Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.843215168Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.843232616Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.843250475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198} {"msg":"Crucible 79223448-af4f-4469-99b8-11f0cf0920fe has session id: dc7f819d-b722-42aa-b11e-0db5cc145638","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.844580125Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Upstairs opts: Upstairs UUID: 79223448-af4f-4469-99b8-11f0cf0920fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.844661766Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Crucible stats registered with UUID: 79223448-af4f-4469-99b8-11f0cf0920fe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.844681114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:12:59.844694783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846181127Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846216333Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198} {"msg":"79223448-af4f-4469-99b8-11f0cf0920fe active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846284126Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846302104Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"0","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846894822Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"0","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846918119Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"1","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846933368Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"1","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846946606Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"2","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846969934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"2","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.846991662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.847015819Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.847040976Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"ds_connection connected from Ok(127.0.0.1:36618)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.847155434Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"ds_connection connected from Ok(127.0.0.1:49899)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.84719461Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"ds_connection connected from Ok(127.0.0.1:48582)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.847220278Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.848567336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"0","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.848593703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"1","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.848612961Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"2","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.849758541Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"2","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.849787748Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"1","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.84995576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"0","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.850737038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.850759966Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.850944936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.850965514Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.850979972Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.850993441Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.85100696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851021258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851034687Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851047725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851064874Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851077792Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851090641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851103239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851115778Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"":"downstairs","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851128747Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851140466Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"0","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851153674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"1","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851166463Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"client":"2","":"downstairs","session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851179391Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.85119126Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"79223448-af4f-4469-99b8-11f0cf0920fe is now active with session: dc7f819d-b722-42aa-b11e-0db5cc145638","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851203849Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851215598Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198,"session_id":"dc7f819d-b722-42aa-b11e-0db5cc145638"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851227337Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198} {"msg":"Activated sub_volume 79223448-af4f-4469-99b8-11f0cf0920fe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:12:59.851423656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1198} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:03.967069637Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.011323958Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.012001187Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.012037733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.012053931Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.012071819Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200} {"msg":"Crucible eea3631f-710e-44e6-a150-3ccd0b5595da has session id: ba8bedab-2e21-486e-955b-bbc589d789b8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.012608993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Upstairs opts: Upstairs UUID: eea3631f-710e-44e6-a150-3ccd0b5595da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.012656478Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Crucible stats registered with UUID: eea3631f-710e-44e6-a150-3ccd0b5595da","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.012685515Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:04.012714821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013165674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013190291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200} {"msg":"eea3631f-710e-44e6-a150-3ccd0b5595da active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013243456Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013269603Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"0","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.01329895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"0","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013314098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"1","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013328257Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"1","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013341716Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013356314Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013369553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013383001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.01339827Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:50852)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013533735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:04.013575161Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63620)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013602448Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:04.013635475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:04.013666641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.013696688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.01415538Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"0","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.014273798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"1","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.014615832Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"0","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.014735399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"1","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.015141746Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:04.015247546Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.014172432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:34250)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.014256893Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.015328421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.015835167Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016406758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016443854Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016464232Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.01648134Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016498518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016516016Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016532504Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016548852Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016565961Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016581569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016597707Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016613376Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016629114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"":"downstairs","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016645313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016661081Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"0","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016677769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"1","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016696267Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"client":"2","":"downstairs","session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T15:13:14.016712825Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016739153Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"eea3631f-710e-44e6-a150-3ccd0b5595da is now active with session: ba8bedab-2e21-486e-955b-bbc589d789b8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016754991Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016769569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200,"session_id":"ba8bedab-2e21-486e-955b-bbc589d789b8"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016784408Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200} {"msg":"Activated sub_volume eea3631f-710e-44e6-a150-3ccd0b5595da","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:14.016799976Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 547 len 16384 data: 2 2 2 2 02/30 Read block 510 len 20480 03/30 Write block 322 len 40960 data: 2 2 2 2 2 2 2 2 2 2 04/30 Write block 598 len 4096 data: 2 05/30 Flush 06/30 Write block 448 len 16384 data: 2 2 2 2 07/30 Write block 182 len 28672 data: 2 2 2 2 2 2 2 08/30 Write block 51 len 40960 data: 2 2 2 2 2 2 2 2 2 2 09/30 Read block 61 len 20480 10/30 Read block 563 len 12288 11/30 Write block 336 len 4096 data: 2 12/30 Read block 97 len 12288 13/30 Read block 447 len 36864 14/30 Read block 488 len 28672 15/30 Read block 442 len 36864 16/30 Write block 324 len 36864 data: 3 3 3 3 3 3 3 3 2 17/30 Read block 179 len 24576 18/30 Flush 19/30 Write block 387 len 40960 data: 2 2 2 2 2 2 2 2 2 2 20/30 Read block 214 len 16384 21/30 Write block 68 len 8192 data: 2 2 22/30 Write block 172 len 4096 data: 2 23/30 Write block 77 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Read block 199 len 4096 25/30 Read block 581 len 40960 26/30 Read block 153 len 8192 27/30 Write block 159 len 24576 data: 2 2 2 2 2 2 28/30 Read block 171 len 8192 29/30 Write block 212 len 8192 data: 2 2 30/30 Write block 595 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 10 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:14.038048154Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1200} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 2 2 2 3 3 3 T T T 4 080-099 1 1 1 1 1 1 T T T 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 T T T 8 160-179 1 1 1 1 1 1 T T T 9 180-199 2 2 2 3 3 3 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 T T T 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 3 3 3 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 T T T Max gen: 2, Max flush: 3 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 F F F 3 060-079 3 3 3 4 4 4 F F F 4 080-099 3 3 3 4 4 4 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 3 3 3 4 4 4 F F F 9 180-199 2 2 2 3 3 3 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 3 3 3 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 3 3 3 4 4 4 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.290412702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.3342229Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.335181079Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.335215766Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.335229654Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.335243893Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208} {"msg":"Crucible 98739257-4392-4654-b8e5-27bc506b2609 has session id: ebb67af9-3a9d-41bc-82c9-7f042fee48fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.335844549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Upstairs opts: Upstairs UUID: 98739257-4392-4654-b8e5-27bc506b2609, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.335884795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Crucible stats registered with UUID: 98739257-4392-4654-b8e5-27bc506b2609","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.335910893Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:18.335928621Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336497581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336526428Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208} {"msg":"98739257-4392-4654-b8e5-27bc506b2609 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336607669Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336625598Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"0","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336657244Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"0","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336680202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"1","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336707129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"1","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336773912Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336813128Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336837525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336858933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336880471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:36673)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336904038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:44988)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.336926276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:18.336955803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:18.33698602Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:18.337105497Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.337138724Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.33764933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"0","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.337676967Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"1","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.338452056Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"1","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.338483132Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"0","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.339057042Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:18.33916814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.338050819Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43883)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.338196193Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.339285449Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.339877377Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 3, 4, 4, 1, 1, 4, 4, 3, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340520029Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[0]R generation[0..12]: [1, 1, 2, 3, 3, 1, 1, 3, 3, 2, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340541187Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340555985Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 3, 4, 4, 1, 1, 4, 4, 3, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340569244Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[1]R generation[0..12]: [1, 1, 2, 3, 3, 1, 1, 3, 3, 2, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340589022Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340603381Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 3, 4, 4, 1, 1, 4, 4, 3, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340616709Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[2]R generation[0..12]: [1, 1, 2, 3, 3, 1, 1, 3, 3, 2, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340629698Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340644626Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340657475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340670124Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340682652Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:13:28.340695121Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"":"downstairs","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340712739Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340724368Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"0","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340737257Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"1","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340750005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"client":"2","":"downstairs","session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340762694Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340774503Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"98739257-4392-4654-b8e5-27bc506b2609 is now active with session: ebb67af9-3a9d-41bc-82c9-7f042fee48fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340789861Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.34080152Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208,"session_id":"ebb67af9-3a9d-41bc-82c9-7f042fee48fb"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340813749Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208} {"msg":"Activated sub_volume 98739257-4392-4654-b8e5-27bc506b2609","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:28.340824877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 581 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 481 len 24576 03/30 Flush 04/30 Write block 292 len 20480 data: 2 2 2 2 2 05/30 Read block 292 len 40960 06/30 Read block 370 len 4096 07/30 Read block 103 len 24576 08/30 Read block 133 len 36864 09/30 Read block 402 len 16384 10/30 Read block 527 len 28672 11/30 Flush 12/30 Write block 66 len 8192 data: 2 2 13/30 Read block 456 len 4096 14/30 Read block 187 len 8192 15/30 Read block 171 len 36864 16/30 Write block 172 len 40960 data: 3 2 2 2 2 2 2 2 2 2 17/30 Read block 340 len 4096 18/30 Write block 284 len 24576 data: 2 2 2 2 2 2 19/30 Read block 417 len 40960 20/30 Write block 57 len 16384 data: 3 3 3 3 21/30 Read block 383 len 8192 22/30 Read block 345 len 20480 23/30 Read block 415 len 8192 24/30 Read block 174 len 12288 25/30 Write block 279 len 40960 data: 2 2 2 2 2 3 3 3 3 3 26/30 Read block 261 len 16384 27/30 Write block 466 len 20480 data: 2 2 2 2 2 28/30 Write block 282 len 32768 data: 3 3 4 4 4 4 4 3 29/30 Write block 534 len 28672 data: 2 2 2 2 2 2 2 30/30 Write block 12 len 4096 data: 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 2 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 3 3 10 Done 15 16 16 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:28.361188948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T T T 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 3 3 3 T T T 3 060-079 3 3 3 4 4 4 T T T 4 080-099 3 3 3 4 4 4 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 3 3 3 4 4 4 T T T 9 180-199 2 2 2 3 3 3 T T T 10 200-219 3 3 3 4 4 4 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 T T T 14 280-299 4 4 4 6 6 6 T T T 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 3 3 3 F F F 23 460-479 1 1 1 1 1 1 T T T 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 1 1 1 1 1 T T T 27 540-559 2 2 2 2 2 2 T T T 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 4, Max flush: 6 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 7 7 7 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 5 5 5 7 7 7 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 3 3 3 4 4 4 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 5 5 5 7 7 7 F F F 9 180-199 5 5 5 7 7 7 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 5 5 5 7 7 7 F F F 14 280-299 5 5 5 7 7 7 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 3 3 3 F F F 23 460-479 5 5 5 7 7 7 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 5 5 5 F F F Max gen: 5, Max flush: 7 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.621510351Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.666992343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.667670121Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.667705098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.667719216Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.667733475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216} {"msg":"Crucible 0e9a249e-e975-4fa0-85ba-b19cbd72489a has session id: c7308e6b-7482-4817-8bbe-739ddc17df01","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.668198366Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Upstairs opts: Upstairs UUID: 0e9a249e-e975-4fa0-85ba-b19cbd72489a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.668236612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Crucible stats registered with UUID: 0e9a249e-e975-4fa0-85ba-b19cbd72489a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.6682586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:32.668334272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.668812541Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.668844908Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216} {"msg":"0e9a249e-e975-4fa0-85ba-b19cbd72489a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.668898692Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.66892417Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"0","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.668953797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"0","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.668968405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.668986703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669004411Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"2","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669029059Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"2","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669052476Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669075524Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669099561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"ds_connection connected from Ok(127.0.0.1:38235)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669124109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"ds_connection connected from Ok(127.0.0.1:38066)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669149316Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:32.669190642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:32.669223109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:32.669247896Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669431836Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.66986903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"0","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.669898147Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"2","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.670527291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"0","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.670556248Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"2","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.671150896Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:32.671175903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.670097917Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"ds_connection connected from Ok(127.0.0.1:38388)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.6702536Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.671328138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.671941593Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[0]R flush_numbers[0..12]: [7, 1, 7, 7, 4, 1, 1, 4, 7, 7, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672695584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[0]R generation[0..12]: [5, 1, 5, 5, 3, 1, 1, 3, 5, 5, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672716622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.67273242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[1]R flush_numbers[0..12]: [7, 1, 7, 7, 4, 1, 1, 4, 7, 7, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672746298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[1]R generation[0..12]: [5, 1, 5, 5, 3, 1, 1, 3, 5, 5, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672759467Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672773006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[2]R flush_numbers[0..12]: [7, 1, 7, 7, 4, 1, 1, 4, 7, 7, 4, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672791504Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[2]R generation[0..12]: [5, 1, 5, 5, 3, 1, 1, 3, 5, 5, 3, 1]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672805432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672818971Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672831449Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672844298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672856827Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672869765Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"":"downstairs","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672882424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672894253Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"0","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672907272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"1","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.67292005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"client":"2","":"downstairs","session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672932889Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672944778Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"0e9a249e-e975-4fa0-85ba-b19cbd72489a is now active with session: c7308e6b-7482-4817-8bbe-739ddc17df01","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672957166Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672969025Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216,"session_id":"c7308e6b-7482-4817-8bbe-739ddc17df01"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:42.672981054Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216} {"msg":"Activated sub_volume 0e9a249e-e975-4fa0-85ba-b19cbd72489a","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2025-03-18T15:13:42.673006231Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 279 len 28672 data: 3 3 3 4 4 5 5 02/30 Read block 155 len 8192 03/30 Write block 15 len 12288 data: 2 2 2 04/30 Read block 379 len 20480 05/30 Read block 477 len 8192 06/30 Read block 83 len 40960 07/30 Read block 496 len 32768 08/30 Read block 472 len 24576 09/30 Write block 85 len 16384 data: 3 3 2 2 10/30 Write block 586 len 16384 data: 3 3 2 2 11/30 Write block 268 len 32768 data: 2 2 2 2 2 2 2 2 12/30 Read block 295 len 28672 13/30 Read block 173 len 20480 14/30 Read block 83 len 16384 15/30 Read block 180 len 20480 16/30 Read block 163 len 12288 17/30 Read block 349 len 12288 18/30 Read block 202 len 8192 19/30 Read block 455 len 24576 20/30 Flush 21/30 Write block 455 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Read block 220 len 40960 23/30 Write block 459 len 4096 data: 3 24/30 Read block 284 len 40960 25/30 Flush 26/30 Write block 454 len 4096 data: 2 27/30 Write block 231 len 40960 data: 2 2 2 2 2 2 2 2 2 2 28/30 Read block 571 len 12288 29/30 Read block 274 len 40960 30/30 Read block 119 len 16384 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:42.694657397Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 5 5 5 7 7 7 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 6 6 6 8 8 8 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 5 5 5 7 7 7 F F F 9 180-199 5 5 5 7 7 7 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 1 1 1 1 1 1 T T T 12 240-259 1 1 1 1 1 1 T T T 13 260-279 6 6 6 8 8 8 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 6 6 6 9 9 9 T T T 23 460-479 6 6 6 9 9 9 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 6 6 6 8 8 8 F F F Max gen: 6, Max flush: 9 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 8 8 8 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 5 5 5 7 7 7 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 6 6 6 8 8 8 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 5 5 5 7 7 7 F F F 9 180-199 5 5 5 7 7 7 F F F 10 200-219 3 3 3 4 4 4 F F F 11 220-239 7 7 7 10 10 10 F F F 12 240-259 7 7 7 10 10 10 F F F 13 260-279 6 6 6 8 8 8 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 7 7 7 10 10 10 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 6 6 6 8 8 8 F F F Max gen: 7, Max flush: 10 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.922334523Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.96748096Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.968232611Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.968275676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.968293224Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.968310913Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224} {"msg":"Crucible ce5b12e9-a96e-4d95-a03a-c12f1976b754 has session id: cecec83e-64dc-4b33-904e-a2e570e32ae3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.96890204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Upstairs opts: Upstairs UUID: ce5b12e9-a96e-4d95-a03a-c12f1976b754, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.968947706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Crucible stats registered with UUID: ce5b12e9-a96e-4d95-a03a-c12f1976b754","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.968972133Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:56.968992721Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969467171Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969492269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224} {"msg":"ce5b12e9-a96e-4d95-a03a-c12f1976b754 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969559451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.96957756Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"0","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969606097Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"0","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969620035Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"1","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969633703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"1","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969646352Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"2","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969660461Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"2","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969673459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969686898Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969714885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64226)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969741712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"ds_connection connected from Ok(127.0.0.1:53590)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.96985908Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"ds_connection connected from Ok(127.0.0.1:52603)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.969916954Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.970536569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"1","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.970573445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"0","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.971158214Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"0","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.971338895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"1","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.971778268Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.971871859Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.97204678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"2","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.972568965Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"2","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[0]R flush_numbers[0..12]: [8, 1, 7, 7, 8, 1, 1, 4, 7, 7, 4, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.97309396Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[0]R generation[0..12]: [6, 1, 5, 5, 6, 1, 1, 3, 5, 5, 3, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973113258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973128107Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[1]R flush_numbers[0..12]: [8, 1, 7, 7, 8, 1, 1, 4, 7, 7, 4, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973148224Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[1]R generation[0..12]: [6, 1, 5, 5, 6, 1, 1, 3, 5, 5, 3, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973172462Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973199959Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[2]R flush_numbers[0..12]: [8, 1, 7, 7, 8, 1, 1, 4, 7, 7, 4, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973218837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[2]R generation[0..12]: [6, 1, 5, 5, 6, 1, 1, 3, 5, 5, 3, 7]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973240565Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973266982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973292379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973312477Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973325496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973341354Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"":"downstairs","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2025-03-18T15:13:56.973371081Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973394259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"0","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973417496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"1","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973432535Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"client":"2","":"downstairs","session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973445633Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.97346996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"ce5b12e9-a96e-4d95-a03a-c12f1976b754 is now active with session: cecec83e-64dc-4b33-904e-a2e570e32ae3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973496258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973517006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224,"session_id":"cecec83e-64dc-4b33-904e-a2e570e32ae3"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973530584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224} {"msg":"Activated sub_volume ce5b12e9-a96e-4d95-a03a-c12f1976b754","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:13:56.973548782Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 29 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 99 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Write block 406 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Read block 231 len 12288 06/30 Flush 07/30 Write block 456 len 4096 data: 3 08/30 Read block 98 len 40960 09/30 Read block 192 len 24576 10/30 Write block 248 len 36864 data: 2 2 2 2 2 2 2 2 2 11/30 Read block 258 len 4096 12/30 Write block 228 len 20480 data: 2 2 2 3 3 13/30 Flush 14/30 Write block 272 len 24576 data: 3 3 3 3 2 2 15/30 Write block 131 len 16384 data: 2 2 2 2 16/30 Read block 593 len 4096 17/30 Flush 18/30 Write block 467 len 40960 data: 3 3 3 3 2 2 2 2 2 2 19/30 Read block 328 len 40960 20/30 Write block 274 len 20480 data: 4 4 3 3 2 21/30 Write block 426 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Write block 362 len 40960 data: 2 2 2 2 2 2 2 2 2 2 23/30 Read block 18 len 32768 24/30 Flush 25/30 Write block 282 len 12288 data: 5 5 6 26/30 Write block 211 len 4096 data: 2 27/30 Read block 327 len 20480 28/30 Write block 470 len 36864 data: 4 3 3 3 3 3 3 2 2 29/30 Write block 163 len 32768 data: 3 3 2 2 2 2 2 2 30/30 Read block 73 len 24576 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 6 4 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:13:56.994857284Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 8 8 8 F F F 1 020-039 8 8 8 11 11 11 F F F 2 040-059 5 5 5 7 7 7 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 8 8 8 12 12 12 F F F 5 100-119 8 8 8 12 12 12 F F F 6 120-139 8 8 8 14 14 14 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 5 5 5 7 7 7 T T T 9 180-199 5 5 5 7 7 7 F F F 10 200-219 3 3 3 4 4 4 T T T 11 220-239 8 8 8 13 13 13 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 8 8 8 15 15 15 F F F 14 280-299 6 6 6 8 8 8 T T T 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 8 8 8 15 15 15 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 8 8 8 12 12 12 F F F 21 420-439 8 8 8 15 15 15 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 8 8 8 15 15 15 T T T 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 6 6 6 8 8 8 F F F Max gen: 8, Max flush: 15 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 8 8 8 F F F 1 020-039 8 8 8 11 11 11 F F F 2 040-059 5 5 5 7 7 7 F F F 3 060-079 5 5 5 7 7 7 F F F 4 080-099 8 8 8 12 12 12 F F F 5 100-119 8 8 8 12 12 12 F F F 6 120-139 8 8 8 14 14 14 F F F 7 140-159 3 3 3 4 4 4 F F F 8 160-179 9 9 9 16 16 16 F F F 9 180-199 5 5 5 7 7 7 F F F 10 200-219 9 9 9 16 16 16 F F F 11 220-239 8 8 8 13 13 13 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 8 8 8 15 15 15 F F F 14 280-299 9 9 9 16 16 16 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 8 8 8 15 15 15 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 8 8 8 12 12 12 F F F 21 420-439 8 8 8 15 15 15 F F F 22 440-459 8 8 8 13 13 13 F F F 23 460-479 9 9 9 16 16 16 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 6 6 6 8 8 8 F F F Max gen: 9, Max flush: 16 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.229462072Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.273535792Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.274172475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.274205522Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.27421954Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.274234079Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232} {"msg":"Crucible 49b63d7f-f6f8-4b87-b390-11928836f71b has session id: f6657f7d-bd29-4ea6-95ff-64eee3b49999","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.274879141Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Upstairs opts: Upstairs UUID: 49b63d7f-f6f8-4b87-b390-11928836f71b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.274922037Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Crucible stats registered with UUID: 49b63d7f-f6f8-4b87-b390-11928836f71b","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.274950294Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:11.274976851Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275371169Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275397347Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232} {"msg":"49b63d7f-f6f8-4b87-b390-11928836f71b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275481738Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275502736Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"0","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275544641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"0","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275579297Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"1","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275608074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"1","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275681877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"2","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275720213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"2","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.27574807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275773437Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275799094Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"ds_connection connected from Ok(127.0.0.1:33215)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275825942Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"ds_connection connected from Ok(127.0.0.1:56514)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275949869Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"ds_connection connected from Ok(127.0.0.1:35882)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.275987555Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.276639266Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"1","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.276673103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"2","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.276689201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"0","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.27736564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"0","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.277397117Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"2","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.277425054Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"1","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278037439Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278060537Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[0]R flush_numbers[0..12]: [8, 11, 7, 7, 12, 12, 14, 4, 16, 7, 16, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278101163Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[0]R generation[0..12]: [6, 8, 5, 5, 8, 8, 8, 3, 9, 5, 9, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278118241Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278132549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[1]R flush_numbers[0..12]: [8, 11, 7, 7, 12, 12, 14, 4, 16, 7, 16, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278146078Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[1]R generation[0..12]: [6, 8, 5, 5, 8, 8, 8, 3, 9, 5, 9, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278159277Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278173115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[2]R flush_numbers[0..12]: [8, 11, 7, 7, 12, 12, 14, 4, 16, 7, 16, 13]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278186564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[2]R generation[0..12]: [6, 8, 5, 5, 8, 8, 8, 3, 9, 5, 9, 8]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278199612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278213211Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278225759Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278238798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278251357Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278267185Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"":"downstairs","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278280034Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:14:11.278291783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"0","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278355556Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"1","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278372204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"client":"2","":"downstairs","session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278385053Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278396912Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"49b63d7f-f6f8-4b87-b390-11928836f71b is now active with session: f6657f7d-bd29-4ea6-95ff-64eee3b49999","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.27840937Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278421129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232,"session_id":"f6657f7d-bd29-4ea6-95ff-64eee3b49999"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278433088Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232} {"msg":"Activated sub_volume 49b63d7f-f6f8-4b87-b390-11928836f71b","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:11.278443976Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 223 len 8192 data: 2 2 02/30 Read block 108 len 40960 03/30 Write block 387 len 4096 data: 3 04/30 Read block 552 len 8192 05/30 Write block 295 len 40960 data: 3 3 2 2 2 2 2 2 2 2 06/30 Write block 566 len 4096 data: 2 07/30 Read block 276 len 24576 08/30 Write block 30 len 4096 data: 3 09/30 Flush 10/30 Write block 451 len 28672 data: 3 2 2 3 3 4 3 11/30 Write block 572 len 8192 data: 2 2 12/30 Write block 266 len 40960 data: 2 2 3 3 3 3 4 4 5 5 13/30 Write block 348 len 8192 data: 2 2 14/30 Flush 15/30 Write block 307 len 12288 data: 2 2 2 16/30 Write block 77 len 32768 data: 3 3 3 3 3 3 3 3 17/30 Write block 183 len 20480 data: 3 3 3 3 3 18/30 Read block 537 len 40960 19/30 Write block 300 len 8192 data: 3 3 20/30 Flush 21/30 Write block 279 len 4096 data: 4 22/30 Write block 138 len 4096 data: 2 23/30 Read block 288 len 20480 24/30 Read block 327 len 40960 25/30 Flush 26/30 Write block 268 len 24576 data: 4 4 4 4 5 5 27/30 Read block 332 len 28672 28/30 Write block 143 len 24576 data: 2 2 2 2 2 2 29/30 Write block 383 len 24576 data: 2 2 2 2 4 3 30/30 Write block 37 len 24576 data: 3 3 2 2 2 2 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:11.295629201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 8 8 8 F F F 1 020-039 10 10 10 17 17 17 T T T 2 040-059 5 5 5 7 7 7 T T T 3 060-079 10 10 10 19 19 19 F F F 4 080-099 10 10 10 19 19 19 F F F 5 100-119 8 8 8 12 12 12 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 3 3 3 4 4 4 T T T 8 160-179 9 9 9 16 16 16 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 9 9 9 16 16 16 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 10 10 10 20 20 20 T T T 14 280-299 10 10 10 17 17 17 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 8 8 8 15 15 15 F F F 19 380-399 10 10 10 17 17 17 T T T 20 400-419 8 8 8 12 12 12 F F F 21 420-439 8 8 8 15 15 15 F F F 22 440-459 10 10 10 18 18 18 F F F 23 460-479 9 9 9 16 16 16 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 10 10 10 18 18 18 F F F 29 580-599 6 6 6 8 8 8 F F F Max gen: 10, Max flush: 20 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 8 8 8 F F F 1 020-039 11 11 11 21 21 21 F F F 2 040-059 11 11 11 21 21 21 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 10 10 10 19 19 19 F F F 5 100-119 8 8 8 12 12 12 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 9 9 9 16 16 16 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 9 9 9 16 16 16 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 11 11 11 21 21 21 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 10 10 10 18 18 18 F F F 18 360-379 8 8 8 15 15 15 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 8 8 8 12 12 12 F F F 21 420-439 8 8 8 15 15 15 F F F 22 440-459 10 10 10 18 18 18 F F F 23 460-479 9 9 9 16 16 16 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 10 10 10 18 18 18 F F F 29 580-599 6 6 6 8 8 8 F F F Max gen: 11, Max flush: 21 Begin loop 6 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.534585608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.579280932Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.579927744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.579962471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.579976529Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.579990808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240} {"msg":"Crucible 64e85781-8072-477d-878a-f09ce202082c has session id: 4f1ca38d-3351-4255-897d-9864743e0c7a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.580552769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Upstairs opts: Upstairs UUID: 64e85781-8072-477d-878a-f09ce202082c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.580599724Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Crucible stats registered with UUID: 64e85781-8072-477d-878a-f09ce202082c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.580629111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:15.580649959Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581165325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581189512Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240} {"msg":"64e85781-8072-477d-878a-f09ce202082c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581259564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581277353Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"0","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.5813059Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"0","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581319618Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"1","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581333487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"1","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581346286Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"2","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581364044Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"2","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581380572Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.58139423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581407479Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52233)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581545094Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34214)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581658343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:40953)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.581703628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.582166959Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"2","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.582338561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"0","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.582752968Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"2","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.582890683Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"0","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.583402819Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.583475742Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.584280937Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"1","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.584791804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"1","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[0]R flush_numbers[0..12]: [8, 21, 21, 19, 19, 12, 20, 21, 16, 19, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585295571Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[0]R generation[0..12]: [6, 11, 11, 10, 10, 8, 10, 11, 9, 10, 9, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585315118Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585329327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[1]R flush_numbers[0..12]: [8, 21, 21, 19, 19, 12, 20, 21, 16, 19, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585343735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[1]R generation[0..12]: [6, 11, 11, 10, 10, 8, 10, 11, 9, 10, 9, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585365543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585391481Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[2]R flush_numbers[0..12]: [8, 21, 21, 19, 19, 12, 20, 21, 16, 19, 16, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585417258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[2]R generation[0..12]: [6, 11, 11, 10, 10, 8, 10, 11, 9, 10, 9, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585442655Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2025-03-18T15:14:15.585473412Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585506129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585527336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585549564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585573421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"":"downstairs","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.58559022Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585611147Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"0","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585635625Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"1","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585656503Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"client":"2","":"downstairs","session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:14:15.58568034Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585704608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"64e85781-8072-477d-878a-f09ce202082c is now active with session: 4f1ca38d-3351-4255-897d-9864743e0c7a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585725895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585745713Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240,"session_id":"4f1ca38d-3351-4255-897d-9864743e0c7a"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585766221Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240} {"msg":"Activated sub_volume 64e85781-8072-477d-878a-f09ce202082c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:15.585784809Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 171 len 20480 data: 2 4 3 3 3 02/30 Read block 7 len 36864 03/30 Read block 33 len 40960 04/30 Flush 05/30 Write block 584 len 8192 data: 3 3 06/30 Read block 316 len 4096 07/30 Write block 18 len 16384 data: 2 2 2 2 08/30 Read block 223 len 20480 09/30 Write block 344 len 8192 data: 2 2 10/30 Write block 410 len 4096 data: 3 11/30 Write block 459 len 16384 data: 4 3 3 3 12/30 Write block 29 len 40960 data: 3 4 3 3 3 3 3 3 4 4 13/30 Write block 214 len 12288 data: 2 2 2 14/30 Read block 319 len 20480 15/30 Read block 314 len 36864 16/30 Write block 503 len 4096 data: 2 17/30 Write block 429 len 8192 data: 3 3 18/30 Read block 15 len 36864 19/30 Read block 518 len 4096 20/30 Read block 62 len 16384 21/30 Read block 442 len 20480 22/30 Write block 446 len 16384 data: 2 2 3 3 23/30 Write block 241 len 20480 data: 2 2 2 2 2 24/30 Flush 25/30 Write block 270 len 40960 data: 5 5 6 6 6 6 4 4 3 5 26/30 Read block 341 len 20480 27/30 Write block 363 len 12288 data: 3 3 3 28/30 Write block 408 len 24576 data: 3 3 4 3 3 3 29/30 Read block 132 len 4096 30/30 Read block 157 len 28672 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 5 5 6 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:15.605094891Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 23 23 23 F F F 1 020-039 12 12 12 23 23 23 F F F 2 040-059 11 11 11 21 21 21 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 10 10 10 19 19 19 F F F 5 100-119 8 8 8 12 12 12 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 12 12 12 22 22 22 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 12 12 12 23 23 23 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 12 12 12 23 23 23 F F F 13 260-279 11 11 11 21 21 21 T T T 14 280-299 10 10 10 17 17 17 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 12 12 12 23 23 23 F F F 18 360-379 8 8 8 15 15 15 T T T 19 380-399 11 11 11 21 21 21 F F F 20 400-419 12 12 12 23 23 23 T T T 21 420-439 12 12 12 23 23 23 F F F 22 440-459 12 12 12 23 23 23 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 12 12 12 23 23 23 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 10 10 10 18 18 18 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 12, Max flush: 23 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 23 23 23 F F F 1 020-039 12 12 12 23 23 23 F F F 2 040-059 11 11 11 21 21 21 F F F 3 060-079 10 10 10 19 19 19 F F F 4 080-099 10 10 10 19 19 19 F F F 5 100-119 8 8 8 12 12 12 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 11 11 11 21 21 21 F F F 8 160-179 12 12 12 22 22 22 F F F 9 180-199 10 10 10 19 19 19 F F F 10 200-219 12 12 12 23 23 23 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 12 12 12 23 23 23 F F F 13 260-279 13 13 13 24 24 24 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 12 12 12 23 23 23 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 13 13 13 24 24 24 F F F 21 420-439 12 12 12 23 23 23 F F F 22 440-459 12 12 12 23 23 23 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 12 12 12 23 23 23 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 10 10 10 18 18 18 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 13, Max flush: 24 Begin loop 7 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.833303418Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:29.87565314Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.87602716Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.907170138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.907777545Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.907810461Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.90782387Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.907838078Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248} {"msg":"Crucible 91ff7878-b9ad-49b7-b012-9d5b8ff675a1 has session id: 2665cca7-bb7b-4c83-8cea-1425045a33b5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.908522216Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Upstairs opts: Upstairs UUID: 91ff7878-b9ad-49b7-b012-9d5b8ff675a1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.908554953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Crucible stats registered with UUID: 91ff7878-b9ad-49b7-b012-9d5b8ff675a1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.908571271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:29.90858487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.90896591Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.908991987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248} {"msg":"91ff7878-b9ad-49b7-b012-9d5b8ff675a1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909079758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909098616Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"0","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909125863Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"0","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909140202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"1","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.90915429Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"1","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909167988Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"2","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909188606Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"2","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909201945Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909220393Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909239381Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60221)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909495934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:50886)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.90953298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57175)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.909560207Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.910058325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"1","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.910230457Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"0","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.910267793Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"2","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.910635234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"1","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.910917475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"2","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911045081Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"0","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911214153Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911403424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[0]R flush_numbers[0..12]: [23, 23, 21, 19, 19, 12, 20, 21, 22, 19, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911561467Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[0]R generation[0..12]: [12, 12, 11, 10, 10, 8, 10, 11, 12, 10, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911589674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911610252Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[1]R flush_numbers[0..12]: [23, 23, 21, 19, 19, 12, 20, 21, 22, 19, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.91162483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[1]R generation[0..12]: [12, 12, 11, 10, 10, 8, 10, 11, 12, 10, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911638379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911652087Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[2]R flush_numbers[0..12]: [23, 23, 21, 19, 19, 12, 20, 21, 22, 19, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911665606Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[2]R generation[0..12]: [12, 12, 11, 10, 10, 8, 10, 11, 12, 10, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911679015Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911692623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911705262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911717851Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911730409Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911742928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"":"downstairs","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911755727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:14:29.911767446Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"0","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911788463Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"1","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911803152Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"client":"2","":"downstairs","session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.91181616Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911828189Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"91ff7878-b9ad-49b7-b012-9d5b8ff675a1 is now active with session: 2665cca7-bb7b-4c83-8cea-1425045a33b5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911840608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911852097Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248,"session_id":"2665cca7-bb7b-4c83-8cea-1425045a33b5"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911863885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248} {"msg":"Activated sub_volume 91ff7878-b9ad-49b7-b012-9d5b8ff675a1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:29.911874984Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 557 len 24576 data: 2 2 2 2 2 2 02/30 Write block 133 len 16384 data: 3 3 2 2 03/30 Read block 350 len 36864 04/30 Write block 347 len 32768 data: 2 3 3 2 2 2 2 2 05/30 Read block 309 len 16384 06/30 Write block 154 len 36864 data: 2 2 2 2 2 3 3 3 3 07/30 Read block 558 len 12288 08/30 Read block 550 len 32768 09/30 Read block 13 len 24576 10/30 Write block 101 len 40960 data: 3 3 3 3 3 3 2 2 2 2 11/30 Write block 194 len 4096 data: 2 12/30 Write block 40 len 8192 data: 3 3 13/30 Read block 340 len 24576 14/30 Read block 387 len 40960 15/30 Write block 430 len 36864 data: 4 3 3 3 3 3 2 2 2 16/30 Read block 564 len 40960 17/30 Read block 286 len 32768 18/30 Write block 2 len 4096 data: 2 19/30 Read block 22 len 36864 20/30 Read block 15 len 32768 21/30 Write block 137 len 4096 data: 2 22/30 Read block 255 len 8192 23/30 Read block 259 len 28672 24/30 Read block 17 len 20480 25/30 Write block 321 len 4096 data: 2 26/30 Write block 93 len 8192 data: 2 2 27/30 Write block 54 len 40960 data: 3 3 3 4 4 4 4 2 2 2 28/30 Read block 212 len 8192 29/30 Write block 49 len 40960 data: 2 2 3 3 3 4 4 4 5 5 30/30 Write block 166 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:29.931790662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 23 23 23 T T T 1 020-039 12 12 12 23 23 23 F F F 2 040-059 11 11 11 21 21 21 T T T 3 060-079 10 10 10 19 19 19 T T T 4 080-099 10 10 10 19 19 19 T T T 5 100-119 8 8 8 12 12 12 T T T 6 120-139 10 10 10 20 20 20 T T T 7 140-159 11 11 11 21 21 21 T T T 8 160-179 12 12 12 22 22 22 T T T 9 180-199 10 10 10 19 19 19 T T T 10 200-219 12 12 12 23 23 23 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 12 12 12 23 23 23 F F F 13 260-279 13 13 13 24 24 24 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 2 2 2 3 3 3 T T T 17 340-359 12 12 12 23 23 23 T T T 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 13 13 13 24 24 24 F F F 21 420-439 12 12 12 23 23 23 T T T 22 440-459 12 12 12 23 23 23 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 12 12 12 23 23 23 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 T T T 28 560-579 10 10 10 18 18 18 T T T 29 580-599 12 12 12 23 23 23 F F F Max gen: 13, Max flush: 24 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 12 12 12 23 23 23 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 14 14 14 25 25 25 F F F 4 080-099 14 14 14 25 25 25 F F F 5 100-119 14 14 14 25 25 25 F F F 6 120-139 14 14 14 25 25 25 F F F 7 140-159 14 14 14 25 25 25 F F F 8 160-179 14 14 14 25 25 25 F F F 9 180-199 14 14 14 25 25 25 F F F 10 200-219 12 12 12 23 23 23 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 12 12 12 23 23 23 F F F 13 260-279 13 13 13 24 24 24 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 10 10 10 19 19 19 F F F 16 320-339 14 14 14 25 25 25 F F F 17 340-359 14 14 14 25 25 25 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 13 13 13 24 24 24 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 12 12 12 23 23 23 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 12 12 12 23 23 23 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 14, Max flush: 25 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.24497731Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.289110954Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.289750387Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.289795342Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.28981226Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.289830609Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256} {"msg":"Crucible e70eb28e-911b-44b8-9527-1024247f96dc has session id: ef5a8ea2-9f85-4904-9d97-0526ef3a56cb","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.290326917Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Upstairs opts: Upstairs UUID: e70eb28e-911b-44b8-9527-1024247f96dc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.290363602Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Crucible stats registered with UUID: e70eb28e-911b-44b8-9527-1024247f96dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.29038987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:34.290411528Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.290853241Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.290884658Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256} {"msg":"e70eb28e-911b-44b8-9527-1024247f96dc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.290954891Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.290978918Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"0","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291007015Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"0","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291022013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"1","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291036352Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"1","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291049561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291063799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291076948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291091356Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291113634Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:57677)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291233271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:34.291274307Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:34.291309133Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:34.29134433Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291506422Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:59949)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.291555217Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.29200943Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"1","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.292036177Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"0","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.292700757Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"0","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.292740503Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"1","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.293362218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:34.293473756Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.292366405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48987)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.292536977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.293975146Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.294525038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[0]R flush_numbers[0..12]: [25, 23, 25, 25, 25, 25, 25, 25, 25, 25, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295115427Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[0]R generation[0..12]: [14, 12, 14, 14, 14, 14, 14, 14, 14, 14, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295136424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295157652Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[1]R flush_numbers[0..12]: [25, 23, 25, 25, 25, 25, 25, 25, 25, 25, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295172361Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[1]R generation[0..12]: [14, 12, 14, 14, 14, 14, 14, 14, 14, 14, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295185829Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295199618Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[2]R flush_numbers[0..12]: [25, 23, 25, 25, 25, 25, 25, 25, 25, 25, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295213246Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[2]R generation[0..12]: [14, 12, 14, 14, 14, 14, 14, 14, 14, 14, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295228085Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295245043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Max found gen is 15","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295260781Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Generation requested: 16 >= found:15","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.29527663Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295292538Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295308186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"":"downstairs","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295324014Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295338433Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"0","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295354621Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"client":"1","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295373299Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","Wait for a query_work_queue command to finish before sending IO pid":1256,"client":"2","":"downstairs","session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295397907Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295413055Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"e70eb28e-911b-44b8-9527-1024247f96dc is now active with session: ef5a8ea2-9f85-4904-9d97-0526ef3a56cb","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295428534Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295443092Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256,"session_id":"ef5a8ea2-9f85-4904-9d97-0526ef3a56cb"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.29545798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256} {"msg":"Activated sub_volume e70eb28e-911b-44b8-9527-1024247f96dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:44.295471379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 95 len 32768 data: 2 2 2 2 3 3 4 4 03/30 Read block 374 len 28672 04/30 Write block 414 len 24576 data: 3 2 2 2 2 2 05/30 Read block 398 len 32768 06/30 Read block 36 len 16384 07/30 Write block 261 len 32768 data: 2 2 2 2 2 3 3 5 08/30 Read block 172 len 40960 09/30 Write block 261 len 20480 data: 3 3 3 3 3 10/30 Write block 515 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Write block 95 len 24576 data: 3 3 3 3 4 4 12/30 Read block 374 len 36864 13/30 Read block 293 len 20480 14/30 Read block 440 len 28672 15/30 Write block 41 len 40960 data: 4 3 2 2 2 2 2 2 3 3 16/30 Read block 548 len 12288 17/30 Flush 18/30 Write block 179 len 36864 data: 3 3 3 3 4 4 4 4 4 19/30 Read block 157 len 16384 20/30 Write block 347 len 36864 data: 3 4 4 3 3 3 3 3 2 21/30 Read block 299 len 8192 22/30 Read block 174 len 12288 23/30 Write block 146 len 36864 data: 3 3 3 2 2 2 2 2 3 24/30 Read block 318 len 40960 25/30 Flush 26/30 Write block 311 len 8192 data: 2 2 27/30 Write block 502 len 16384 data: 2 3 2 2 28/30 Read block 418 len 40960 29/30 Read block 546 len 40960 30/30 Write block 405 len 12288 data: 2 3 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:44.318537936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 12 12 12 23 23 23 F F F 2 040-059 16 16 16 27 27 27 F F F 3 060-079 14 14 14 25 25 25 F F F 4 080-099 16 16 16 27 27 27 F F F 5 100-119 16 16 16 27 27 27 F F F 6 120-139 14 14 14 25 25 25 F F F 7 140-159 16 16 16 28 28 28 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 16 16 16 28 28 28 F F F 10 200-219 12 12 12 23 23 23 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 12 12 12 23 23 23 F F F 13 260-279 16 16 16 27 27 27 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 10 10 10 19 19 19 T T T 16 320-339 14 14 14 25 25 25 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 16 16 16 27 27 27 T T T 21 420-439 14 14 14 25 25 25 F F F 22 440-459 12 12 12 23 23 23 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 16 16 16 27 27 27 T T T 26 520-539 16 16 16 27 27 27 F F F 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 16, Max flush: 28 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 12 12 12 23 23 23 F F F 2 040-059 16 16 16 27 27 27 F F F 3 060-079 14 14 14 25 25 25 F F F 4 080-099 16 16 16 27 27 27 F F F 5 100-119 16 16 16 27 27 27 F F F 6 120-139 14 14 14 25 25 25 F F F 7 140-159 16 16 16 28 28 28 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 16 16 16 28 28 28 F F F 10 200-219 12 12 12 23 23 23 F F F 11 220-239 10 10 10 17 17 17 F F F 12 240-259 12 12 12 23 23 23 F F F 13 260-279 16 16 16 27 27 27 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 14 14 14 25 25 25 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 17 17 17 29 29 29 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 12 12 12 23 23 23 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 17 17 17 29 29 29 F F F 26 520-539 16 16 16 27 27 27 F F F 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 17, Max flush: 29 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.548604908Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.591914938Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.592512945Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.592554301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.592571769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.592589687Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264} {"msg":"Crucible ac6c9fd4-be54-4cde-b181-051ddb9e402f has session id: e47069ec-aefe-4dd6-99a2-12089e305a1a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.59313003Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Upstairs opts: Upstairs UUID: ac6c9fd4-be54-4cde-b181-051ddb9e402f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593168586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Crucible stats registered with UUID: ac6c9fd4-be54-4cde-b181-051ddb9e402f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593191324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:48.593211712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593717388Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593743346Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264} {"msg":"ac6c9fd4-be54-4cde-b181-051ddb9e402f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593818748Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593848405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"0","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593878162Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"0","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593896Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"1","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593912368Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"1","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593925597Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593942435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.593966372Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.59399319Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.594018027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:48.59408188Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:48.594128485Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56742)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.594157222Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57528)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.59418307Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:48.594213716Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.594236994Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.594763709Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"0","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.594803404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"1","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.595412201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"1","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.595554435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"0","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.596019437Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:48.596123496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.59563154Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47828)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.595971304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.596890098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.597614051Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[0]R flush_numbers[0..12]: [25, 23, 27, 25, 27, 27, 25, 28, 28, 28, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598310868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[0]R generation[0..12]: [14, 12, 16, 14, 16, 16, 14, 16, 16, 16, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598335475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598355253Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[1]R flush_numbers[0..12]: [25, 23, 27, 25, 27, 27, 25, 28, 28, 28, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598372612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[1]R generation[0..12]: [14, 12, 16, 14, 16, 16, 14, 16, 16, 16, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.59838918Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598406258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[2]R flush_numbers[0..12]: [25, 23, 27, 25, 27, 27, 25, 28, 28, 28, 23, 17]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598428426Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[2]R generation[0..12]: [14, 12, 16, 14, 16, 16, 14, 16, 16, 16, 12, 10]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598445094Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598461802Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598477351Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598493049Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598508767Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598530635Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"":"downstairs","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598546673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598561032Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"0","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.59857708Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"1","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598592819Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"client":"2","":"downstairs","session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598608807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598623555Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"ac6c9fd4-be54-4cde-b181-051ddb9e402f is now active with session: e47069ec-aefe-4dd6-99a2-12089e305a1a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598639034Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.598653582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264,"session_id":"e47069ec-aefe-4dd6-99a2-12089e305a1a"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:14:58.59867077Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264} {"msg":"Activated sub_volume ac6c9fd4-be54-4cde-b181-051ddb9e402f","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:14:58.598684609Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 213 len 20480 data: 3 3 3 3 2 02/30 Read block 347 len 8192 03/30 Write block 231 len 16384 data: 4 4 3 3 04/30 Write block 401 len 32768 data: 2 2 2 2 3 4 4 4 05/30 Read block 309 len 20480 06/30 Write block 239 len 32768 data: 3 3 3 3 3 3 3 2 07/30 Write block 139 len 40960 data: 2 2 2 2 3 3 3 4 4 4 08/30 Write block 526 len 12288 data: 2 2 2 09/30 Read block 570 len 8192 10/30 Read block 464 len 20480 11/30 Write block 324 len 4096 data: 4 12/30 Write block 492 len 24576 data: 2 2 2 2 2 2 13/30 Read block 360 len 20480 14/30 Write block 107 len 4096 data: 3 15/30 Read block 46 len 36864 16/30 Read block 98 len 28672 17/30 Read block 293 len 32768 18/30 Write block 338 len 32768 data: 2 2 2 2 2 2 3 3 19/30 Write block 429 len 36864 data: 4 5 4 4 4 4 4 3 3 20/30 Read block 564 len 28672 21/30 Read block 186 len 12288 22/30 Read block 535 len 32768 23/30 Flush 24/30 Write block 185 len 8192 data: 5 5 25/30 Read block 313 len 8192 26/30 Write block 31 len 12288 data: 4 4 4 27/30 Write block 517 len 24576 data: 3 3 3 3 3 3 28/30 Read block 508 len 24576 29/30 Write block 58 len 32768 data: 6 5 5 3 3 3 2 2 30/30 Write block 534 len 24576 data: 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:14:58.619177856Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 12 12 12 23 23 23 T T T 2 040-059 16 16 16 27 27 27 T T T 3 060-079 14 14 14 25 25 25 T T T 4 080-099 16 16 16 27 27 27 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 18 18 18 30 30 30 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 16 16 16 28 28 28 T T T 10 200-219 18 18 18 30 30 30 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 16 16 16 27 27 27 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 18 18 18 30 30 30 F F F 17 340-359 18 18 18 30 30 30 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 18 18 18 30 30 30 F F F 21 420-439 18 18 18 30 30 30 F F F 22 440-459 12 12 12 23 23 23 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 18 18 18 30 30 30 F F F 25 500-519 17 17 17 29 29 29 T T T 26 520-539 18 18 18 30 30 30 T T T 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 18, Max flush: 30 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 19 19 19 31 31 31 F F F 2 040-059 19 19 19 31 31 31 F F F 3 060-079 19 19 19 31 31 31 F F F 4 080-099 16 16 16 27 27 27 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 18 18 18 30 30 30 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 19 19 19 31 31 31 F F F 10 200-219 18 18 18 30 30 30 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 16 16 16 27 27 27 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 18 18 18 30 30 30 F F F 17 340-359 18 18 18 30 30 30 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 18 18 18 30 30 30 F F F 21 420-439 18 18 18 30 30 30 F F F 22 440-459 12 12 12 23 23 23 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 18 18 18 30 30 30 F F F 25 500-519 19 19 19 31 31 31 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 19, Max flush: 31 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.861125549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.905892986Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.906509401Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.906543507Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.906558086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.906576684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272} {"msg":"Crucible fc513ffd-ca18-49a2-b04b-4e60584463fe has session id: 4055090a-30db-4508-86fc-916c322c8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907120527Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Upstairs opts: Upstairs UUID: fc513ffd-ca18-49a2-b04b-4e60584463fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907162362Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Crucible stats registered with UUID: fc513ffd-ca18-49a2-b04b-4e60584463fe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.90718772Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:02.907209327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.90775656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907787337Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272} {"msg":"fc513ffd-ca18-49a2-b04b-4e60584463fe active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907843751Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907868548Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"0","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907896445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"0","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907910544Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"1","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907924542Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"1","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907938751Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"2","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907964248Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"2","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.907996435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.908025681Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.908051999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"ds_connection connected from Ok(127.0.0.1:52791)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.908077506Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"ds_connection connected from Ok(127.0.0.1:47019)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.908109803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"ds_connection connected from Ok(127.0.0.1:56159)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.90813972Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.908838456Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"1","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.908877872Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"0","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.909513035Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"1","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.909541522Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"0","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.90956324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"2","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910236059Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910263186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910276795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"2","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[0]R flush_numbers[0..12]: [25, 31, 31, 31, 27, 30, 30, 30, 28, 31, 30, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910790201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[0]R generation[0..12]: [14, 19, 19, 19, 16, 18, 18, 18, 16, 19, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910818598Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910833866Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[1]R flush_numbers[0..12]: [25, 31, 31, 31, 27, 30, 30, 30, 28, 31, 30, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910847925Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[1]R generation[0..12]: [14, 19, 19, 19, 16, 18, 18, 18, 16, 19, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910861514Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910874962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[2]R flush_numbers[0..12]: [25, 31, 31, 31, 27, 30, 30, 30, 28, 31, 30, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910887941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[2]R generation[0..12]: [14, 19, 19, 19, 16, 18, 18, 18, 16, 19, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.91090103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910915158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910928047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910940885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910953514Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910966103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"":"downstairs","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910979051Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.910991Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"0","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.911006128Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"1","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.911019077Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"client":"2","":"downstairs","session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.911031936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.911043784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"Wait for a query_work_queue command to finish before sending IO 4055090a-30db-4508-86fc-916c322c8830"} {"msg":"fc513ffd-ca18-49a2-b04b-4e60584463fe is now active with session: 4055090a-30db-4508-86fc-916c322c8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.911062833Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.911074731Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272,"session_id":"4055090a-30db-4508-86fc-916c322c8830"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.91108658Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272} {"msg":"Activated sub_volume fc513ffd-ca18-49a2-b04b-4e60584463fe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:02.911097419Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 20480 data: 2 2 2 2 2 02/30 Read block 305 len 36864 03/30 Read block 268 len 4096 04/30 Write block 120 len 40960 data: 2 2 2 2 2 2 2 2 2 2 05/30 Write block 333 len 32768 data: 2 2 2 3 2 3 3 3 06/30 Read block 231 len 12288 07/30 Flush 08/30 Write block 438 len 12288 data: 3 2 2 09/30 Flush 10/30 Flush 11/30 Write block 431 len 20480 data: 5 5 5 5 5 12/30 Read block 582 len 36864 13/30 Read block 277 len 12288 14/30 Read block 544 len 8192 15/30 Write block 247 len 4096 data: 2 16/30 Flush 17/30 Write block 36 len 12288 data: 4 5 5 18/30 Read block 100 len 4096 19/30 Read block 286 len 40960 20/30 Write block 416 len 20480 data: 3 3 3 3 2 21/30 Flush 22/30 Write block 191 len 8192 data: 2 2 23/30 Write block 337 len 28672 data: 3 4 4 4 3 3 3 24/30 Write block 186 len 32768 data: 6 5 3 2 2 3 3 2 25/30 Flush 26/30 Write block 268 len 20480 data: 6 5 6 6 7 27/30 Read block 170 len 40960 28/30 Read block 559 len 28672 29/30 Read block 552 len 12288 30/30 Write block 191 len 12288 data: 4 4 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:02.930549885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 20 20 20 36 36 36 F F F 2 040-059 19 19 19 31 31 31 F F F 3 060-079 20 20 20 32 32 32 F F F 4 080-099 16 16 16 27 27 27 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 20 20 20 37 37 37 T T T 10 200-219 18 18 18 30 30 30 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 16 16 16 27 27 27 T T T 14 280-299 10 10 10 17 17 17 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 20 20 20 36 36 36 F F F 21 420-439 20 20 20 36 36 36 F F F 22 440-459 20 20 20 33 33 33 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 18 18 18 30 30 30 F F F 25 500-519 19 19 19 31 31 31 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 20, Max flush: 37 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 20 20 20 36 36 36 F F F 2 040-059 19 19 19 31 31 31 F F F 3 060-079 20 20 20 32 32 32 F F F 4 080-099 16 16 16 27 27 27 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 21 21 21 38 38 38 F F F 10 200-219 18 18 18 30 30 30 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 21 21 21 38 38 38 F F F 14 280-299 10 10 10 17 17 17 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 11 11 11 21 21 21 F F F 20 400-419 20 20 20 36 36 36 F F F 21 420-439 20 20 20 36 36 36 F F F 22 440-459 20 20 20 33 33 33 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 18 18 18 30 30 30 F F F 25 500-519 19 19 19 31 31 31 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 12 12 12 23 23 23 F F F Max gen: 21, Max flush: 38 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.150264284Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:07.193746576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.194117797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.225844994Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.226385887Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.226419614Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.226434382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.226449471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280} {"msg":"Crucible de6b184d-b908-46f7-b40a-a104589852c1 has session id: b31d8983-2987-4c34-81a5-72e8a70e6710","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227111881Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Upstairs opts: Upstairs UUID: de6b184d-b908-46f7-b40a-a104589852c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227142788Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Crucible stats registered with UUID: de6b184d-b908-46f7-b40a-a104589852c1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227157886Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:07.227170405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227534877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227557454Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280} {"msg":"de6b184d-b908-46f7-b40a-a104589852c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227607239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227632406Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"0","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227664893Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"0","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227679581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"1","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.22769412Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"1","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227735706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"2","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227770892Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"2","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227798469Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227820966Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227835355Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"ds_connection connected from Ok(127.0.0.1:35802)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227849344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"ds_connection connected from Ok(127.0.0.1:42031)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.227865602Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"ds_connection connected from Ok(127.0.0.1:55198)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.228128204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.228638971Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"2","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.228681736Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"1","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.228909292Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"0","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.229327228Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"1","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.229354925Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"2","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.229371044Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"0","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.229897668Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230005347Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[0]R flush_numbers[0..12]: [25, 36, 31, 32, 27, 30, 32, 30, 28, 38, 30, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230079219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[0]R generation[0..12]: [14, 20, 19, 20, 16, 18, 20, 18, 16, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230100487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230115685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[1]R flush_numbers[0..12]: [25, 36, 31, 32, 27, 30, 32, 30, 28, 38, 30, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230129604Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[1]R generation[0..12]: [14, 20, 19, 20, 16, 18, 20, 18, 16, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230142873Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230156481Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[2]R flush_numbers[0..12]: [25, 36, 31, 32, 27, 30, 32, 30, 28, 38, 30, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.23017057Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[2]R generation[0..12]: [14, 20, 19, 20, 16, 18, 20, 18, 16, 21, 18, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230184048Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230197447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:15:07.230214345Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230239772Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.23026438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230289187Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"":"downstairs","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230314535Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230338382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"0","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230365089Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"1","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Transition from WaitQuorum to Active"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230396386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"client":"2","":"downstairs","session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230422583Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230435832Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"de6b184d-b908-46f7-b40a-a104589852c1 is now active with session: b31d8983-2987-4c34-81a5-72e8a70e6710","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.23044945Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230461219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280,"session_id":"b31d8983-2987-4c34-81a5-72e8a70e6710"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230473098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280} {"msg":"Activated sub_volume de6b184d-b908-46f7-b40a-a104589852c1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:07.230484007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 396 len 8192 data: 3 2 02/30 Write block 583 len 4096 data: 3 03/30 Write block 415 len 32768 data: 3 4 4 4 4 3 2 2 04/30 Flush 05/30 Write block 82 len 4096 data: 4 06/30 Write block 490 len 8192 data: 2 2 07/30 Write block 443 len 40960 data: 2 2 2 3 3 4 4 3 4 3 08/30 Flush 09/30 Write block 500 len 16384 data: 2 2 3 4 10/30 Read block 139 len 8192 11/30 Read block 305 len 36864 12/30 Write block 191 len 40960 data: 5 5 4 3 2 2 2 2 2 2 13/30 Read block 5 len 36864 14/30 Write block 270 len 24576 data: 7 7 8 7 7 7 15/30 Read block 180 len 12288 16/30 Read block 415 len 8192 17/30 Read block 434 len 16384 18/30 Write block 501 len 8192 data: 3 4 19/30 Read block 162 len 28672 20/30 Read block 395 len 16384 21/30 Write block 58 len 40960 data: 7 6 6 4 4 4 3 3 3 3 22/30 Read block 576 len 12288 23/30 Read block 467 len 8192 24/30 Read block 303 len 40960 25/30 Read block 237 len 12288 26/30 Read block 589 len 12288 27/30 Read block 580 len 28672 28/30 Read block 97 len 12288 29/30 Write block 283 len 36864 data: 6 7 6 5 5 5 4 2 2 30/30 Read block 393 len 40960 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 4 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 22 22 20 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:07.250771735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 20 20 20 36 36 36 F F F 2 040-059 19 19 19 31 31 31 T T T 3 060-079 20 20 20 32 32 32 T T T 4 080-099 22 22 22 40 40 40 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 21 21 21 38 38 38 T T T 10 200-219 18 18 18 30 30 30 T T T 11 220-239 18 18 18 30 30 30 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 21 21 21 38 38 38 T T T 14 280-299 10 10 10 17 17 17 T T T 15 300-319 17 17 17 29 29 29 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 22 22 22 39 39 39 F F F 20 400-419 22 22 22 39 39 39 F F F 21 420-439 22 22 22 39 39 39 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 22 22 22 40 40 40 F F F 25 500-519 19 19 19 31 31 31 T T T 26 520-539 19 19 19 31 31 31 F F F 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 22, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 25 25 25 F F F 1 020-039 20 20 20 36 36 36 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 23 23 23 41 41 41 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 23 23 23 41 41 41 F F F 11 220-239 18 18 18 30 30 30 F F F 12 240-259 20 20 20 35 35 35 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 23 23 23 41 41 41 F F F 15 300-319 17 17 17 29 29 29 F F F 16 320-339 20 20 20 37 37 37 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 13 13 13 24 24 24 F F F 19 380-399 22 22 22 39 39 39 F F F 20 400-419 22 22 22 39 39 39 F F F 21 420-439 22 22 22 39 39 39 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 22 22 22 40 40 40 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 19 19 19 31 31 31 F F F 27 540-559 14 14 14 25 25 25 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.528033489Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.571705241Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.572441424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.57247523Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.572489369Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.572503817Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288} {"msg":"Crucible 4db6bd86-24bf-487f-a0b8-8a341c4c1a22 has session id: 81accd1d-c951-4dc8-b453-ce4d50abd65c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573111424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Upstairs opts: Upstairs UUID: 4db6bd86-24bf-487f-a0b8-8a341c4c1a22, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.57314863Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Crucible stats registered with UUID: 4db6bd86-24bf-487f-a0b8-8a341c4c1a22","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573165578Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:11.573178246Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573642428Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573665115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288} {"msg":"4db6bd86-24bf-487f-a0b8-8a341c4c1a22 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573742697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573768735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"0","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573798331Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"0","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.57381296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"1","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573832238Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"1","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573855725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573881523Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573898041Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573925238Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.573952355Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33200)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.574013829Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38384)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.574047875Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:11.574317507Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:11.574358213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:11.574389619Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.574450243Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.574699837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"0","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.574736963Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"1","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.575343029Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"1","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.575383235Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"0","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.575946376Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:11.575972793Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.575528792Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56698)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.575656848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.57687594Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.577477787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[0]R flush_numbers[0..12]: [25, 36, 41, 41, 40, 30, 32, 30, 28, 41, 41, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578084263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[0]R generation[0..12]: [14, 20, 23, 23, 22, 18, 20, 18, 16, 23, 23, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578105941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.57812123Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[1]R flush_numbers[0..12]: [25, 36, 41, 41, 40, 30, 32, 30, 28, 41, 41, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578135108Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[1]R generation[0..12]: [14, 20, 23, 23, 22, 18, 20, 18, 16, 23, 23, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578148476Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578162105Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[2]R flush_numbers[0..12]: [25, 36, 41, 41, 40, 30, 32, 30, 28, 41, 41, 30]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578175464Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[2]R generation[0..12]: [14, 20, 23, 23, 22, 18, 20, 18, 16, 23, 23, 18]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578188572Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578202151Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.5782146Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578233728Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578246576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578259145Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"":"downstairs","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578271754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578283522Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"0","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578296401Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"1","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.57830899Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"client":"2","":"downstairs","session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578321808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578333567Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"4db6bd86-24bf-487f-a0b8-8a341c4c1a22 is now active with session: 81accd1d-c951-4dc8-b453-ce4d50abd65c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578346086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:15:21.578357725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288,"session_id":"81accd1d-c951-4dc8-b453-ce4d50abd65c"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578376553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288} {"msg":"Activated sub_volume 4db6bd86-24bf-487f-a0b8-8a341c4c1a22","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:21.578387981Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 3 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 319 len 32768 03/30 Write block 365 len 12288 data: 4 3 3 04/30 Write block 300 len 8192 data: 4 4 05/30 Flush 06/30 Write block 26 len 24576 data: 2 2 2 4 5 5 07/30 Write block 484 len 24576 data: 2 2 2 2 2 2 08/30 Flush 09/30 Write block 70 len 12288 data: 3 3 3 10/30 Read block 48 len 8192 11/30 Read block 272 len 40960 12/30 Read block 500 len 36864 13/30 Read block 475 len 12288 14/30 Write block 532 len 40960 data: 2 2 4 4 4 4 4 4 3 2 15/30 Read block 114 len 32768 16/30 Read block 585 len 8192 17/30 Read block 373 len 12288 18/30 Read block 151 len 4096 19/30 Read block 364 len 24576 20/30 Write block 54 len 16384 data: 5 5 5 6 21/30 Read block 169 len 36864 22/30 Write block 239 len 12288 data: 4 4 4 23/30 Flush 24/30 Write block 219 len 36864 data: 2 2 2 2 3 3 2 2 2 25/30 Write block 416 len 16384 data: 5 5 5 5 26/30 Write block 400 len 12288 data: 2 3 3 27/30 Write block 307 len 36864 data: 3 3 3 2 3 3 2 2 2 28/30 Write block 321 len 16384 data: 3 3 3 5 29/30 Read block 465 len 4096 30/30 Read block 19 len 28672 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:21.598805157Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 F F F 1 020-039 24 24 24 43 43 43 F F F 2 040-059 24 24 24 44 44 44 F F F 3 060-079 24 24 24 44 44 44 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 23 23 23 41 41 41 T T T 11 220-239 24 24 24 44 44 44 T T T 12 240-259 24 24 24 44 44 44 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 23 23 23 41 41 41 F F F 15 300-319 24 24 24 42 42 42 T T T 16 320-339 20 20 20 37 37 37 T T T 17 340-359 20 20 20 37 37 37 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 22 22 22 39 39 39 F F F 20 400-419 22 22 22 39 39 39 T T T 21 420-439 22 22 22 39 39 39 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 24 24 24 43 43 43 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 24 24 24 44 44 44 F F F 27 540-559 24 24 24 44 44 44 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 24, Max flush: 44 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 F F F 1 020-039 24 24 24 43 43 43 F F F 2 040-059 24 24 24 44 44 44 F F F 3 060-079 24 24 24 44 44 44 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 25 25 25 45 45 45 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 24 24 24 44 44 44 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 23 23 23 41 41 41 F F F 15 300-319 25 25 25 45 45 45 F F F 16 320-339 25 25 25 45 45 45 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 24 24 24 42 42 42 F F F 19 380-399 22 22 22 39 39 39 F F F 20 400-419 25 25 25 45 45 45 F F F 21 420-439 22 22 22 39 39 39 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 12 12 12 23 23 23 F F F 24 480-499 24 24 24 43 43 43 F F F 25 500-519 23 23 23 41 41 41 F F F 26 520-539 24 24 24 44 44 44 F F F 27 540-559 24 24 24 44 44 44 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 25, Max flush: 45 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.839310024Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:35.882436843Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.88285077Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.914103786Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.914716792Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.914757607Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.914775576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.914793204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296} {"msg":"Crucible bda8ae11-a214-4156-a64f-91fbfa8fdaf1 has session id: f165b05f-43af-440c-8bc2-9d7b039c9ed9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.915419508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Upstairs opts: Upstairs UUID: bda8ae11-a214-4156-a64f-91fbfa8fdaf1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.915459184Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Crucible stats registered with UUID: bda8ae11-a214-4156-a64f-91fbfa8fdaf1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.915488411Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:35.915503789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.915958821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.915983269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296} {"msg":"bda8ae11-a214-4156-a64f-91fbfa8fdaf1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916062881Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916079879Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"0","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916107656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"0","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916121994Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"1","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916135853Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"1","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916148652Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"2","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.91616236Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"2","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916175299Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916189197Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916209625Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38621)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916389616Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58508)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.91644301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37742)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.916587205Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.917222828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"1","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.917357874Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"2","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.917504979Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"0","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.917787209Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"1","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.917988558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"2","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.91816548Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"0","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918330062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918528831Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[0]R flush_numbers[0..12]: [42, 43, 44, 44, 40, 30, 32, 30, 28, 41, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918738509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[0]R generation[0..12]: [24, 24, 24, 24, 22, 18, 20, 18, 16, 23, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918758087Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918773426Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[1]R flush_numbers[0..12]: [42, 43, 44, 44, 40, 30, 32, 30, 28, 41, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918787884Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[1]R generation[0..12]: [24, 24, 24, 24, 22, 18, 20, 18, 16, 23, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918806692Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918821601Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[2]R flush_numbers[0..12]: [42, 43, 44, 44, 40, 30, 32, 30, 28, 41, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918835899Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[2]R generation[0..12]: [24, 24, 24, 24, 22, 18, 20, 18, 16, 23, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918849748Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918864696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918878395Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918891103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:15:35.918904192Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.91892081Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"":"downstairs","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918934259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918946687Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"0","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918960406Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"1","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918973725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"client":"2","":"downstairs","session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918987083Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.918999582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"bda8ae11-a214-4156-a64f-91fbfa8fdaf1 is now active with session: f165b05f-43af-440c-8bc2-9d7b039c9ed9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.91901411Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.919026149Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296,"session_id":"f165b05f-43af-440c-8bc2-9d7b039c9ed9"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.919038188Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296} {"msg":"Activated sub_volume bda8ae11-a214-4156-a64f-91fbfa8fdaf1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:35.919048967Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 415 len 24576 data: 4 6 6 6 6 4 02/30 Write block 25 len 4096 data: 2 03/30 Write block 3 len 20480 data: 3 3 3 3 3 04/30 Write block 430 len 8192 data: 6 6 05/30 Write block 380 len 40960 data: 2 2 2 3 3 3 3 5 4 3 06/30 Read block 75 len 36864 07/30 Read block 271 len 36864 08/30 Read block 547 len 16384 09/30 Read block 563 len 20480 10/30 Read block 388 len 32768 11/30 Write block 365 len 4096 data: 5 12/30 Write block 492 len 36864 data: 3 3 3 3 3 3 2 2 3 13/30 Read block 199 len 12288 14/30 Read block 46 len 8192 15/30 Read block 237 len 16384 16/30 Read block 414 len 20480 17/30 Read block 33 len 40960 18/30 Write block 256 len 16384 data: 3 2 2 2 19/30 Write block 475 len 12288 data: 4 4 3 20/30 Write block 471 len 16384 data: 4 4 4 4 21/30 Read block 471 len 12288 22/30 Write block 417 len 24576 data: 7 7 7 5 3 3 23/30 Read block 391 len 16384 24/30 Read block 272 len 24576 25/30 Read block 1 len 16384 26/30 Read block 279 len 12288 27/30 Write block 404 len 32768 data: 3 4 5 5 5 4 5 4 28/30 Read block 30 len 28672 29/30 Read block 417 len 20480 30/30 Write block 504 len 32768 data: 3 3 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:35.939160964Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 24 24 24 42 42 42 T T T 1 020-039 24 24 24 43 43 43 T T T 2 040-059 24 24 24 44 44 44 F F F 3 060-079 24 24 24 44 44 44 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 25 25 25 45 45 45 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 24 24 24 44 44 44 T T T 13 260-279 23 23 23 41 41 41 F F F 14 280-299 23 23 23 41 41 41 F F F 15 300-319 25 25 25 45 45 45 F F F 16 320-339 25 25 25 45 45 45 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 24 24 24 42 42 42 T T T 19 380-399 22 22 22 39 39 39 T T T 20 400-419 25 25 25 45 45 45 T T T 21 420-439 22 22 22 39 39 39 T T T 22 440-459 22 22 22 40 40 40 F F F 23 460-479 12 12 12 23 23 23 T T T 24 480-499 24 24 24 43 43 43 T T T 25 500-519 23 23 23 41 41 41 T T T 26 520-539 24 24 24 44 44 44 F F F 27 540-559 24 24 24 44 44 44 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 25, Max flush: 45 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 46 46 46 F F F 1 020-039 26 26 26 46 46 46 F F F 2 040-059 24 24 24 44 44 44 F F F 3 060-079 24 24 24 44 44 44 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 23 23 23 41 41 41 F F F 10 200-219 25 25 25 45 45 45 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 26 26 26 46 46 46 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 23 23 23 41 41 41 F F F 15 300-319 25 25 25 45 45 45 F F F 16 320-339 25 25 25 45 45 45 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 26 26 26 46 46 46 F F F 19 380-399 26 26 26 46 46 46 F F F 20 400-419 26 26 26 46 46 46 F F F 21 420-439 26 26 26 46 46 46 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 26 26 26 46 46 46 F F F 24 480-499 26 26 26 46 46 46 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 24 24 24 44 44 44 F F F 27 540-559 24 24 24 44 44 44 F F F 28 560-579 14 14 14 25 25 25 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 26, Max flush: 46 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.232595918Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:40.275045949Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.275403721Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.306239381Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.306816761Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.306848788Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.306862566Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.306876794Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304} {"msg":"Crucible 2e6ce2ef-ea6f-4495-a651-9522090f25fa has session id: 5316abb0-562a-4c8b-8be1-c605490f5e63","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.307449485Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Upstairs opts: Upstairs UUID: 2e6ce2ef-ea6f-4495-a651-9522090f25fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.307547764Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Crucible stats registered with UUID: 2e6ce2ef-ea6f-4495-a651-9522090f25fa","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.307567642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:40.307580671Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308120534Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308148921Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304} {"msg":"2e6ce2ef-ea6f-4495-a651-9522090f25fa active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308216974Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308233982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"0","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308262349Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"0","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308276498Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"1","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308290366Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"1","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308303325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"2","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308332362Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"2","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.30834732Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308369178Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308384606Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"ds_connection connected from Ok(127.0.0.1:48549)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308494795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"ds_connection connected from Ok(127.0.0.1:35909)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.30853368Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"ds_connection connected from Ok(127.0.0.1:37684)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.308668196Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.309261204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"1","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.309291981Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"2","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.309368763Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"0","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.309905646Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"1","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310047492Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"2","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.31015675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"0","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310513643Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310605633Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[0]R flush_numbers[0..12]: [46, 46, 44, 44, 40, 30, 32, 30, 28, 41, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310717311Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[0]R generation[0..12]: [26, 26, 24, 24, 22, 18, 20, 18, 16, 23, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310736859Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310751327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[1]R flush_numbers[0..12]: [46, 46, 44, 44, 40, 30, 32, 30, 28, 41, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310764806Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[1]R generation[0..12]: [26, 26, 24, 24, 22, 18, 20, 18, 16, 23, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310778015Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310791403Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[2]R flush_numbers[0..12]: [46, 46, 44, 44, 40, 30, 32, 30, 28, 41, 45, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310805142Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[2]R generation[0..12]: [26, 26, 24, 24, 22, 18, 20, 18, 16, 23, 25, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.31082344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310838758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Max found gen is 27","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310852007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Generation requested: 28 >= found:27","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310864696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310877074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310895622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"":"downstairs","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310908391Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.31092001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"0","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310933148Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"1","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310946867Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"client":"2","":"downstairs","session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310959956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310971814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"2e6ce2ef-ea6f-4495-a651-9522090f25fa is now active with session: 5316abb0-562a-4c8b-8be1-c605490f5e63","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310984173Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.310996632Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304,"session_id":"5316abb0-562a-4c8b-8be1-c605490f5e63"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.311008491Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304} {"msg":"Activated sub_volume 2e6ce2ef-ea6f-4495-a651-9522090f25fa","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:40.311019649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 293 len 12288 data: 3 3 4 02/30 Write block 306 len 8192 data: 2 4 03/30 Flush 04/30 Write block 537 len 40960 data: 5 5 5 4 3 2 2 2 2 2 05/30 Read block 90 len 12288 06/30 Read block 165 len 28672 07/30 Write block 2 len 4096 data: 3 08/30 Read block 553 len 12288 09/30 Write block 110 len 20480 data: 3 2 2 2 2 10/30 Flush 11/30 Write block 362 len 8192 data: 3 4 12/30 Read block 311 len 40960 13/30 Read block 131 len 24576 14/30 Write block 451 len 24576 data: 5 4 3 4 4 5 15/30 Write block 62 len 24576 data: 5 5 4 4 4 4 16/30 Write block 111 len 24576 data: 3 3 3 3 2 2 17/30 Read block 54 len 40960 18/30 Read block 83 len 8192 19/30 Write block 215 len 8192 data: 4 4 20/30 Read block 255 len 24576 21/30 Write block 572 len 28672 data: 3 3 2 2 2 2 2 22/30 Read block 135 len 40960 23/30 Read block 589 len 32768 24/30 Read block 509 len 8192 25/30 Read block 325 len 24576 26/30 Write block 389 len 20480 data: 4 3 3 3 3 27/30 Write block 192 len 20480 data: 6 5 4 3 3 28/30 Read block 324 len 4096 29/30 Read block 322 len 32768 30/30 Write block 217 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 2 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:40.330380096Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 48 48 48 F F F 1 020-039 26 26 26 46 46 46 F F F 2 040-059 24 24 24 44 44 44 F F F 3 060-079 24 24 24 44 44 44 T T T 4 080-099 22 22 22 40 40 40 F F F 5 100-119 28 28 28 48 48 48 T T T 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 23 23 23 41 41 41 T T T 10 200-219 25 25 25 45 45 45 T T T 11 220-239 25 25 25 45 45 45 F F F 12 240-259 26 26 26 46 46 46 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 28 28 28 47 47 47 F F F 16 320-339 25 25 25 45 45 45 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 26 26 26 46 46 46 T T T 19 380-399 26 26 26 46 46 46 T T T 20 400-419 26 26 26 46 46 46 F F F 21 420-439 26 26 26 46 46 46 F F F 22 440-459 22 22 22 40 40 40 T T T 23 460-479 26 26 26 46 46 46 F F F 24 480-499 26 26 26 46 46 46 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 28 28 28 48 48 48 F F F 27 540-559 28 28 28 48 48 48 F F F 28 560-579 14 14 14 25 25 25 T T T 29 580-599 22 22 22 39 39 39 F F F Max gen: 28, Max flush: 48 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 48 48 48 F F F 1 020-039 26 26 26 46 46 46 F F F 2 040-059 24 24 24 44 44 44 F F F 3 060-079 29 29 29 49 49 49 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 29 29 29 49 49 49 F F F 6 120-139 20 20 20 32 32 32 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 29 29 29 49 49 49 F F F 10 200-219 29 29 29 49 49 49 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 26 26 26 46 46 46 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 28 28 28 47 47 47 F F F 16 320-339 25 25 25 45 45 45 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 29 29 29 49 49 49 F F F 19 380-399 29 29 29 49 49 49 F F F 20 400-419 26 26 26 46 46 46 F F F 21 420-439 26 26 26 46 46 46 F F F 22 440-459 29 29 29 49 49 49 F F F 23 460-479 26 26 26 46 46 46 F F F 24 480-499 26 26 26 46 46 46 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 28 28 28 48 48 48 F F F 27 540-559 28 28 28 48 48 48 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 29, Max flush: 49 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.616645483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:44.661452276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.661916448Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.693078014Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.693713047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.693747424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.693767061Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.69378225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312} {"msg":"Crucible d25457b3-4708-4ee9-934c-48799d862380 has session id: 47cf4776-6d31-4f0a-a171-537fd98456bd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.694397255Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Upstairs opts: Upstairs UUID: d25457b3-4708-4ee9-934c-48799d862380, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.694424922Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Crucible stats registered with UUID: d25457b3-4708-4ee9-934c-48799d862380","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.694439191Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:44.69445137Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.694950027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.694973605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312} {"msg":"d25457b3-4708-4ee9-934c-48799d862380 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695057256Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695075834Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"0","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695104521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"0","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695118589Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"1","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695132978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"1","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695151946Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"2","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695167854Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"2","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695181073Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695194961Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.69520826Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:51299)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695402459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38462)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695450205Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:62651)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.695479251Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.696135273Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"2","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.696283397Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"1","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.696598904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"0","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.696678696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"2","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.696707222Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"1","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697181923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"0","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697333407Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697357654Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[0]R flush_numbers[0..12]: [48, 46, 44, 49, 40, 49, 32, 30, 28, 49, 49, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697700768Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[0]R generation[0..12]: [28, 26, 24, 29, 22, 29, 20, 18, 16, 29, 29, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697737194Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697763862Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[1]R flush_numbers[0..12]: [48, 46, 44, 49, 40, 49, 32, 30, 28, 49, 49, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697790289Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[1]R generation[0..12]: [28, 26, 24, 29, 22, 29, 20, 18, 16, 29, 29, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697814356Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697838614Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[2]R flush_numbers[0..12]: [48, 46, 44, 49, 40, 49, 32, 30, 28, 49, 49, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697865061Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[2]R generation[0..12]: [28, 26, 24, 29, 22, 29, 20, 18, 16, 29, 29, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697892628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697920225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697953942Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.697980839Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Next flush: 50","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698005916Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698031843Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"":"downstairs","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.69806394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698088637Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"0","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698114755Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1312,"client":"1","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698144642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"client":"2","":"downstairs","session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698170499Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698186897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"d25457b3-4708-4ee9-934c-48799d862380 is now active with session: 47cf4776-6d31-4f0a-a171-537fd98456bd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698200326Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698212574Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312,"session_id":"47cf4776-6d31-4f0a-a171-537fd98456bd"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698224563Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312} {"msg":"Activated sub_volume d25457b3-4708-4ee9-934c-48799d862380","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:44.698235292Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 335 len 28672 data: 3 4 4 5 5 5 4 02/30 Write block 181 len 16384 data: 4 4 5 5 03/30 Write block 193 len 28672 data: 6 5 4 4 3 3 3 04/30 Read block 226 len 24576 05/30 Flush 06/30 Write block 35 len 4096 data: 4 07/30 Read block 256 len 8192 08/30 Write block 128 len 12288 data: 3 3 2 09/30 Write block 7 len 36864 data: 4 3 3 2 2 3 2 2 3 10/30 Write block 200 len 16384 data: 3 2 2 2 11/30 Write block 425 len 40960 data: 2 3 3 3 5 7 7 6 6 6 12/30 Flush 13/30 Write block 11 len 24576 data: 3 4 3 3 4 3 14/30 Read block 319 len 8192 15/30 Read block 444 len 40960 16/30 Read block 506 len 20480 17/30 Read block 193 len 40960 18/30 Read block 303 len 12288 19/30 Write block 470 len 16384 data: 5 5 5 5 20/30 Read block 370 len 20480 21/30 Read block 458 len 8192 22/30 Read block 430 len 20480 23/30 Write block 308 len 28672 data: 4 4 3 4 4 3 3 24/30 Write block 247 len 40960 data: 3 3 3 3 3 3 3 3 3 4 25/30 Read block 585 len 16384 26/30 Read block 350 len 32768 27/30 Write block 356 len 4096 data: 2 28/30 Read block 438 len 28672 29/30 Read block 473 len 16384 30/30 Read block 558 len 28672 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 6 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 18 17 17 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:44.717923174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 51 51 51 T T T 1 020-039 30 30 30 51 51 51 F F F 2 040-059 24 24 24 44 44 44 F F F 3 060-079 29 29 29 49 49 49 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 29 29 29 49 49 49 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 30 30 30 50 50 50 F F F 10 200-219 30 30 30 51 51 51 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 26 26 26 46 46 46 T T T 13 260-279 23 23 23 41 41 41 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 28 28 28 47 47 47 T T T 16 320-339 30 30 30 50 50 50 F F F 17 340-359 30 30 30 50 50 50 T T T 18 360-379 29 29 29 49 49 49 F F F 19 380-399 29 29 29 49 49 49 F F F 20 400-419 26 26 26 46 46 46 F F F 21 420-439 30 30 30 51 51 51 F F F 22 440-459 29 29 29 49 49 49 F F F 23 460-479 26 26 26 46 46 46 T T T 24 480-499 26 26 26 46 46 46 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 28 28 28 48 48 48 F F F 27 540-559 28 28 28 48 48 48 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 30, Max flush: 51 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 24 24 24 44 44 44 F F F 3 060-079 29 29 29 49 49 49 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 29 29 29 49 49 49 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 F F F 9 180-199 30 30 30 50 50 50 F F F 10 200-219 30 30 30 51 51 51 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 31 31 31 52 52 52 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 31 31 31 52 52 52 F F F 16 320-339 30 30 30 50 50 50 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 29 29 29 49 49 49 F F F 19 380-399 29 29 29 49 49 49 F F F 20 400-419 26 26 26 46 46 46 F F F 21 420-439 30 30 30 51 51 51 F F F 22 440-459 29 29 29 49 49 49 F F F 23 460-479 31 31 31 52 52 52 F F F 24 480-499 26 26 26 46 46 46 F F F 25 500-519 26 26 26 46 46 46 F F F 26 520-539 28 28 28 48 48 48 F F F 27 540-559 28 28 28 48 48 48 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 31, Max flush: 52 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:48.984721027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.029610101Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.030241355Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.030275721Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03028951Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.030303908Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320} {"msg":"Crucible b7dcf0a9-552d-4954-b01a-14febad00c84 has session id: 9f7df62f-6198-4b28-b3cc-432e34323407","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.030957509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Upstairs opts: Upstairs UUID: b7dcf0a9-552d-4954-b01a-14febad00c84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.030997405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Crucible stats registered with UUID: b7dcf0a9-552d-4954-b01a-14febad00c84","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031023172Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:49.0310451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031500372Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03152481Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320} {"msg":"b7dcf0a9-552d-4954-b01a-14febad00c84 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031597952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03161608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"0","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031642947Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"0","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031657226Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"1","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031681063Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"1","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031762675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"2","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03180174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"2","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031826608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031851355Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031879303Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"ds_connection connected from Ok(127.0.0.1:61622)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03190493Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"ds_connection connected from Ok(127.0.0.1:51598)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031926987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"ds_connection connected from Ok(127.0.0.1:53732)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.031964903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.032657741Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"1","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.032788597Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"0","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.033163497Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"1","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.033191734Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"2","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.033212652Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"0","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.033780303Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03380516Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.033825518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"2","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[0]R flush_numbers[0..12]: [52, 51, 44, 49, 40, 49, 51, 30, 28, 50, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034357242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[0]R generation[0..12]: [31, 30, 24, 29, 22, 29, 30, 18, 16, 30, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03438027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034394668Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[1]R flush_numbers[0..12]: [52, 51, 44, 49, 40, 49, 51, 30, 28, 50, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034408707Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[1]R generation[0..12]: [31, 30, 24, 29, 22, 29, 30, 18, 16, 30, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034422445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034436474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[2]R flush_numbers[0..12]: [52, 51, 44, 49, 40, 49, 51, 30, 28, 50, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034454162Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[2]R generation[0..12]: [31, 30, 24, 29, 22, 29, 30, 18, 16, 30, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03446798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034481559Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034494348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034507156Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034586508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034603096Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"":"downstairs","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034615945Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034627684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"0","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034640702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"1","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034653361Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"client":"2","":"downstairs","session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.03466625Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034678388Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"b7dcf0a9-552d-4954-b01a-14febad00c84 is now active with session: 9f7df62f-6198-4b28-b3cc-432e34323407","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034690927Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034702626Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320,"session_id":"9f7df62f-6198-4b28-b3cc-432e34323407"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034717284Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320} {"msg":"Activated sub_volume b7dcf0a9-552d-4954-b01a-14febad00c84","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:15:49.034728413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 173 len 24576 data: 4 4 4 3 3 3 02/30 Write block 196 len 4096 data: 5 03/30 Write block 422 len 4096 data: 4 04/30 Read block 262 len 20480 05/30 Write block 285 len 36864 data: 7 6 6 6 5 3 3 3 4 06/30 Read block 94 len 24576 07/30 Read block 439 len 28672 08/30 Write block 455 len 40960 data: 5 6 4 3 5 4 4 4 3 3 09/30 Write block 523 len 24576 data: 2 2 2 3 3 3 10/30 Read block 579 len 16384 11/30 Read block 132 len 24576 12/30 Read block 374 len 24576 13/30 Write block 510 len 40960 data: 3 3 2 2 2 3 3 4 4 4 14/30 Write block 42 len 20480 data: 4 3 3 3 3 15/30 Write block 435 len 4096 data: 6 16/30 Write block 292 len 28672 data: 4 5 4 5 4 3 3 17/30 Read block 22 len 20480 18/30 Write block 398 len 28672 data: 2 2 3 4 4 3 4 19/30 Write block 330 len 36864 data: 4 4 3 3 3 4 5 5 6 20/30 Read block 150 len 8192 21/30 Read block 92 len 12288 22/30 Write block 314 len 24576 data: 4 3 2 2 2 2 23/30 Write block 292 len 20480 data: 5 6 5 6 5 24/30 Read block 54 len 40960 25/30 Read block 45 len 20480 26/30 Read block 441 len 28672 27/30 Read block 179 len 36864 28/30 Write block 119 len 12288 data: 2 3 3 29/30 Write block 371 len 36864 data: 3 2 2 2 2 2 2 2 2 30/30 Read block 476 len 16384 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 2 2 Done 30 30 28 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:15:49.054912043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 24 24 24 44 44 44 T T T 3 060-079 29 29 29 49 49 49 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 29 29 29 49 49 49 T T T 6 120-139 30 30 30 51 51 51 T T T 7 140-159 18 18 18 30 30 30 F F F 8 160-179 16 16 16 28 28 28 T T T 9 180-199 30 30 30 50 50 50 T T T 10 200-219 30 30 30 51 51 51 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 31 31 31 52 52 52 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 28 28 28 47 47 47 T T T 15 300-319 31 31 31 52 52 52 T T T 16 320-339 30 30 30 50 50 50 T T T 17 340-359 31 31 31 52 52 52 F F F 18 360-379 29 29 29 49 49 49 T T T 19 380-399 29 29 29 49 49 49 T T T 20 400-419 26 26 26 46 46 46 T T T 21 420-439 30 30 30 51 51 51 T T T 22 440-459 29 29 29 49 49 49 T T T 23 460-479 31 31 31 52 52 52 T T T 24 480-499 26 26 26 46 46 46 F F F 25 500-519 26 26 26 46 46 46 T T T 26 520-539 28 28 28 48 48 48 T T T 27 540-559 28 28 28 48 48 48 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 31, Max flush: 52 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 29 29 29 49 49 49 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 32 32 32 53 53 53 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 32 32 32 53 53 53 F F F 9 180-199 32 32 32 53 53 53 F F F 10 200-219 30 30 30 51 51 51 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 31 31 31 52 52 52 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 32 32 32 53 53 53 F F F 15 300-319 32 32 32 53 53 53 F F F 16 320-339 32 32 32 53 53 53 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 32 32 32 53 53 53 F F F 19 380-399 32 32 32 53 53 53 F F F 20 400-419 32 32 32 53 53 53 F F F 21 420-439 32 32 32 53 53 53 F F F 22 440-459 32 32 32 53 53 53 F F F 23 460-479 32 32 32 53 53 53 F F F 24 480-499 26 26 26 46 46 46 F F F 25 500-519 32 32 32 53 53 53 F F F 26 520-539 32 32 32 53 53 53 F F F 27 540-559 28 28 28 48 48 48 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 22 22 22 39 39 39 F F F Max gen: 32, Max flush: 53 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.345482991Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.389546232Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.39023553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.390277105Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.390295543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.390313482Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328} {"msg":"Crucible d5e6df89-32e5-40b4-aff6-0b1d3395328b has session id: 66ec7965-bf33-47eb-8a45-01398221b9b8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.390846885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Upstairs opts: Upstairs UUID: d5e6df89-32e5-40b4-aff6-0b1d3395328b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.390880952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Crucible stats registered with UUID: d5e6df89-32e5-40b4-aff6-0b1d3395328b","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.39089684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:03.390910049Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391365351Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391389458Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328} {"msg":"d5e6df89-32e5-40b4-aff6-0b1d3395328b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391453812Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.39147193Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"0","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391500067Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"0","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391519545Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391537243Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"2","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391586718Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"2","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391611705Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391637742Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.39166109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38591)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391684897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:34468)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.391876407Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:03.391917013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:03.391945Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:03.391974027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.392118672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.392318501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"2","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.392349497Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"0","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.392948734Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"2","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.392976562Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"0","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.393485098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:03.3935651Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.393927495Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45581)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.394049862Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.395264035Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.395885219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[0]R flush_numbers[0..12]: [52, 51, 53, 49, 40, 53, 53, 30, 53, 53, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396508864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[0]R generation[0..12]: [31, 30, 32, 29, 22, 32, 32, 18, 32, 32, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396533521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396549879Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[1]R flush_numbers[0..12]: [52, 51, 53, 49, 40, 53, 53, 30, 53, 53, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396564478Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[1]R generation[0..12]: [31, 30, 32, 29, 22, 32, 32, 18, 32, 32, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396578046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396597384Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[2]R flush_numbers[0..12]: [52, 51, 53, 49, 40, 53, 53, 30, 53, 53, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396612973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[2]R generation[0..12]: [31, 30, 32, 29, 22, 32, 32, 18, 32, 32, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396626791Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.39664024Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396653918Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Generation requested: 34 >= found:33","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396666727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396679266Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396691745Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"":"downstairs","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396704413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396715992Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"0","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396729001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"1","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:16:13.396741749Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"client":"2","":"downstairs","session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396760177Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396772266Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"d5e6df89-32e5-40b4-aff6-0b1d3395328b is now active with session: 66ec7965-bf33-47eb-8a45-01398221b9b8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396784805Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396798743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328,"session_id":"66ec7965-bf33-47eb-8a45-01398221b9b8"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396812432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328} {"msg":"Activated sub_volume d5e6df89-32e5-40b4-aff6-0b1d3395328b","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:13.396823991Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 366 len 8192 data: 4 4 02/30 Write block 292 len 12288 data: 6 7 6 03/30 Read block 560 len 28672 04/30 Flush 05/30 Write block 100 len 20480 data: 5 5 5 4 4 06/30 Write block 9 len 36864 data: 4 3 4 5 4 4 5 4 3 07/30 Write block 415 len 12288 data: 5 7 8 08/30 Read block 488 len 12288 09/30 Read block 53 len 36864 10/30 Read block 151 len 4096 11/30 Write block 581 len 36864 data: 3 3 4 4 4 4 4 3 3 12/30 Write block 174 len 4096 data: 5 13/30 Write block 63 len 8192 data: 6 5 14/30 Read block 145 len 12288 15/30 Flush 16/30 Write block 584 len 32768 data: 5 5 5 5 4 4 2 2 17/30 Write block 384 len 24576 data: 4 4 4 6 5 5 18/30 Read block 161 len 4096 19/30 Read block 432 len 4096 20/30 Read block 444 len 36864 21/30 Write block 478 len 20480 data: 3 2 2 2 2 22/30 Read block 147 len 32768 23/30 Write block 533 len 24576 data: 3 5 5 5 6 6 24/30 Write block 153 len 8192 data: 3 4 25/30 Write block 294 len 28672 data: 7 7 6 4 4 3 5 26/30 Read block 69 len 8192 27/30 Write block 438 len 36864 data: 4 3 3 2 2 3 3 3 4 28/30 Read block 164 len 24576 29/30 Read block 505 len 36864 30/30 Write block 61 len 36864 data: 5 6 7 6 5 5 5 3 3 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:13.416227242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 55 55 55 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 34 34 34 55 55 55 T T T 4 080-099 22 22 22 40 40 40 F F F 5 100-119 34 34 34 55 55 55 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 18 18 18 30 30 30 T T T 8 160-179 34 34 34 55 55 55 F F F 9 180-199 32 32 32 53 53 53 F F F 10 200-219 30 30 30 51 51 51 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 31 31 31 52 52 52 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 34 34 34 54 54 54 T T T 15 300-319 32 32 32 53 53 53 T T T 16 320-339 32 32 32 53 53 53 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 34 34 34 54 54 54 F F F 19 380-399 32 32 32 53 53 53 T T T 20 400-419 34 34 34 55 55 55 F F F 21 420-439 32 32 32 53 53 53 T T T 22 440-459 32 32 32 53 53 53 T T T 23 460-479 32 32 32 53 53 53 T T T 24 480-499 26 26 26 46 46 46 T T T 25 500-519 32 32 32 53 53 53 F F F 26 520-539 32 32 32 53 53 53 T T T 27 540-559 28 28 28 48 48 48 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 34 34 34 55 55 55 T T T Max gen: 34, Max flush: 55 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 55 55 55 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 32 32 32 53 53 53 F F F 3 060-079 35 35 35 56 56 56 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 34 34 34 55 55 55 F F F 6 120-139 32 32 32 53 53 53 F F F 7 140-159 35 35 35 56 56 56 F F F 8 160-179 34 34 34 55 55 55 F F F 9 180-199 32 32 32 53 53 53 F F F 10 200-219 30 30 30 51 51 51 F F F 11 220-239 25 25 25 45 45 45 F F F 12 240-259 31 31 31 52 52 52 F F F 13 260-279 23 23 23 41 41 41 F F F 14 280-299 35 35 35 56 56 56 F F F 15 300-319 35 35 35 56 56 56 F F F 16 320-339 32 32 32 53 53 53 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 34 34 34 54 54 54 F F F 19 380-399 35 35 35 56 56 56 F F F 20 400-419 34 34 34 55 55 55 F F F 21 420-439 35 35 35 56 56 56 F F F 22 440-459 35 35 35 56 56 56 F F F 23 460-479 35 35 35 56 56 56 F F F 24 480-499 35 35 35 56 56 56 F F F 25 500-519 32 32 32 53 53 53 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 28 28 28 48 48 48 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 35 35 35 56 56 56 F F F Max gen: 35, Max flush: 56 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.681734544Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.725899384Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.726551046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.726585672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.72660351Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.726618469Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336} {"msg":"Crucible b6eee158-a5cf-41f2-a76c-1a18bbaad3a0 has session id: 40cad2a2-9671-4ac1-96c4-17ec9b64ec20","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727192019Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Upstairs opts: Upstairs UUID: b6eee158-a5cf-41f2-a76c-1a18bbaad3a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727230285Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Crucible stats registered with UUID: b6eee158-a5cf-41f2-a76c-1a18bbaad3a0","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727246813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:27.727259732Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727694906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727726363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336} {"msg":"b6eee158-a5cf-41f2-a76c-1a18bbaad3a0 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727798915Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727824482Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"0","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727858159Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"0","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727882146Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"1","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727907084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"1","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727935221Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727966827Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.727994235Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.728087405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.728115742Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:27.728144239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:27.728173246Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:27.728200433Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"ds_connection connected from Ok(127.0.0.1:47319)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.72822395Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"ds_connection connected from Ok(127.0.0.1:47137)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.728249638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.728275375Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.728671663Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"0","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.72870758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"1","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.729378729Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"1","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.729419135Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"0","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.729955478Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:27.730106213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.729658191Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"ds_connection connected from Ok(127.0.0.1:58869)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.729746982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.730942036Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.731563691Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[0]R flush_numbers[0..12]: [55, 51, 53, 56, 40, 55, 53, 56, 55, 53, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732159418Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[0]R generation[0..12]: [34, 30, 32, 35, 22, 34, 32, 35, 34, 32, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732183346Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732198834Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[1]R flush_numbers[0..12]: [55, 51, 53, 56, 40, 55, 53, 56, 55, 53, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732213023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[1]R generation[0..12]: [34, 30, 32, 35, 22, 34, 32, 35, 34, 32, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732226731Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.73224074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[2]R flush_numbers[0..12]: [55, 51, 53, 56, 40, 55, 53, 56, 55, 53, 51, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732254588Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[2]R generation[0..12]: [34, 30, 32, 35, 22, 34, 32, 35, 34, 32, 30, 25]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732274706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732289304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732301803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732314322Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Next flush: 57","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.73232664Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732338979Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"":"downstairs","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732352088Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732363627Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"0","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732376965Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"1","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732389804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"client":"2","":"downstairs","session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732402712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732414601Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"b6eee158-a5cf-41f2-a76c-1a18bbaad3a0 is now active with session: 40cad2a2-9671-4ac1-96c4-17ec9b64ec20","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.73242711Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732438769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336,"session_id":"40cad2a2-9671-4ac1-96c4-17ec9b64ec20"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732450678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336} {"msg":"Activated sub_volume b6eee158-a5cf-41f2-a76c-1a18bbaad3a0","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:37.732462156Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 549 len 40960 data: 3 3 2 2 2 2 2 2 3 3 02/30 Write block 138 len 36864 data: 3 3 3 3 3 4 4 4 5 03/30 Write block 257 len 40960 data: 3 3 3 2 4 4 4 4 4 4 04/30 Read block 585 len 36864 05/30 Read block 140 len 20480 06/30 Write block 330 len 12288 data: 5 5 4 07/30 Read block 294 len 32768 08/30 Read block 406 len 12288 09/30 Read block 113 len 12288 10/30 Write block 335 len 12288 data: 5 6 6 11/30 Write block 515 len 16384 data: 4 4 5 5 12/30 Read block 298 len 24576 13/30 Read block 475 len 32768 14/30 Write block 443 len 16384 data: 4 4 4 5 15/30 Read block 19 len 16384 16/30 Write block 505 len 20480 data: 4 3 3 3 3 17/30 Read block 265 len 36864 18/30 Read block 366 len 28672 19/30 Read block 69 len 40960 20/30 Flush 21/30 Write block 147 len 28672 data: 5 5 3 3 3 3 4 22/30 Read block 479 len 28672 23/30 Write block 180 len 16384 data: 4 5 5 6 24/30 Write block 265 len 20480 data: 5 5 4 7 6 25/30 Read block 448 len 24576 26/30 Write block 52 len 4096 data: 4 27/30 Read block 586 len 40960 28/30 Read block 567 len 12288 29/30 Write block 231 len 8192 data: 5 5 30/30 Read block 520 len 16384 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 2 Sent Sent Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 8 8 10 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:37.754092094Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 55 55 55 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 32 32 32 53 53 53 T T T 3 060-079 35 35 35 56 56 56 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 34 34 34 55 55 55 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 36 36 36 57 57 57 T T T 8 160-179 34 34 34 55 55 55 F F F 9 180-199 32 32 32 53 53 53 T T T 10 200-219 30 30 30 51 51 51 F F F 11 220-239 25 25 25 45 45 45 T T T 12 240-259 36 36 36 57 57 57 F F F 13 260-279 36 36 36 57 57 57 T T T 14 280-299 35 35 35 56 56 56 F F F 15 300-319 35 35 35 56 56 56 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 34 34 34 54 54 54 F F F 19 380-399 35 35 35 56 56 56 F F F 20 400-419 34 34 34 55 55 55 F F F 21 420-439 35 35 35 56 56 56 F F F 22 440-459 36 36 36 57 57 57 F F F 23 460-479 35 35 35 56 56 56 F F F 24 480-499 35 35 35 56 56 56 F F F 25 500-519 36 36 36 57 57 57 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 36 36 36 57 57 57 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 35 35 35 56 56 56 F F F Max gen: 36, Max flush: 57 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 55 55 55 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 37 37 37 58 58 58 F F F 3 060-079 35 35 35 56 56 56 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 34 34 34 55 55 55 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 37 37 37 58 58 58 F F F 8 160-179 34 34 34 55 55 55 F F F 9 180-199 37 37 37 58 58 58 F F F 10 200-219 30 30 30 51 51 51 F F F 11 220-239 37 37 37 58 58 58 F F F 12 240-259 36 36 36 57 57 57 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 35 35 35 56 56 56 F F F 15 300-319 35 35 35 56 56 56 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 34 34 34 54 54 54 F F F 19 380-399 35 35 35 56 56 56 F F F 20 400-419 34 34 34 55 55 55 F F F 21 420-439 35 35 35 56 56 56 F F F 22 440-459 36 36 36 57 57 57 F F F 23 460-479 35 35 35 56 56 56 F F F 24 480-499 35 35 35 56 56 56 F F F 25 500-519 36 36 36 57 57 57 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 36 36 36 57 57 57 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 35 35 35 56 56 56 F F F Max gen: 37, Max flush: 58 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:41.995548738Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:42.038474229Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.038881996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.070038823Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.070654098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.070687665Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.070701383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.070720611Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344} {"msg":"Crucible a60cdbd4-c4e0-4a1a-b48a-8ee5ba689b52 has session id: cbe600b3-504c-4c40-bb15-d88a611f6cbe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.071334057Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Upstairs opts: Upstairs UUID: a60cdbd4-c4e0-4a1a-b48a-8ee5ba689b52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.071374702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Crucible stats registered with UUID: a60cdbd4-c4e0-4a1a-b48a-8ee5ba689b52","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.07139485Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:42.071410738Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.071837004Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.071865061Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344} {"msg":"a60cdbd4-c4e0-4a1a-b48a-8ee5ba689b52 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.071915285Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.071934383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"0","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.07196712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"0","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.071984408Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"1","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072001776Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"1","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072018515Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"2","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072032183Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"2","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072048742Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.0720623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072085158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:57825)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072229832Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:43341)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072365078Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"ds_connection connected from Ok(127.0.0.1:34186)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072391366Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072852097Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"1","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.072892563Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"2","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.073037348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"0","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.073524296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"1","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.073558243Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"2","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.07387247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"0","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074190697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074285287Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[0]R flush_numbers[0..12]: [55, 51, 58, 56, 40, 55, 57, 58, 55, 58, 51, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074397505Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[0]R generation[0..12]: [34, 30, 37, 35, 22, 34, 36, 37, 34, 37, 30, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074422262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074437611Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[1]R flush_numbers[0..12]: [55, 51, 58, 56, 40, 55, 57, 58, 55, 58, 51, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074451599Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[1]R generation[0..12]: [34, 30, 37, 35, 22, 34, 36, 37, 34, 37, 30, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074465517Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074479206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[2]R flush_numbers[0..12]: [55, 51, 58, 56, 40, 55, 57, 58, 55, 58, 51, 58]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074492735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[2]R generation[0..12]: [34, 30, 37, 35, 22, 34, 36, 37, 34, 37, 30, 37]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074506283Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074520082Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.07453269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074545499Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074557978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074570397Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"":"downstairs","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074583315Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074594944Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"0","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:16:42.074609982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"1","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.07463004Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"client":"2","":"downstairs","session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074643129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074655038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"a60cdbd4-c4e0-4a1a-b48a-8ee5ba689b52 is now active with session: cbe600b3-504c-4c40-bb15-d88a611f6cbe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074667686Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074679525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344,"session_id":"cbe600b3-504c-4c40-bb15-d88a611f6cbe"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074691504Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344} {"msg":"Activated sub_volume a60cdbd4-c4e0-4a1a-b48a-8ee5ba689b52","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:42.074702443Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 510 len 20480 data: 4 4 3 3 3 03/30 Read block 165 len 28672 04/30 Write block 584 len 20480 data: 6 6 6 6 5 05/30 Write block 410 len 12288 data: 6 5 4 06/30 Read block 185 len 32768 07/30 Flush 08/30 Write block 222 len 8192 data: 3 4 09/30 Write block 32 len 16384 data: 5 5 4 5 10/30 Read block 86 len 24576 11/30 Write block 189 len 36864 data: 3 3 6 7 7 6 5 6 4 12/30 Write block 376 len 40960 data: 3 3 3 3 3 3 3 4 5 5 13/30 Flush 14/30 Write block 460 len 24576 data: 5 5 5 4 4 2 15/30 Read block 427 len 4096 16/30 Read block 37 len 4096 17/30 Read block 228 len 32768 18/30 Write block 509 len 36864 data: 4 5 5 4 4 4 5 5 6 19/30 Write block 10 len 28672 data: 4 5 6 5 5 6 5 20/30 Read block 79 len 32768 21/30 Read block 409 len 24576 22/30 Read block 255 len 24576 23/30 Flush 24/30 Write block 20 len 16384 data: 3 3 2 2 25/30 Write block 200 len 16384 data: 4 3 3 3 26/30 Read block 522 len 36864 27/30 Read block 589 len 8192 28/30 Write block 190 len 28672 data: 4 7 8 8 7 6 7 29/30 Write block 455 len 12288 data: 6 7 5 30/30 Read block 391 len 32768 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 3 4 Done 7 6 4 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:42.095233706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 38 38 38 61 61 61 T T T 2 040-059 37 37 37 58 58 58 F F F 3 060-079 35 35 35 56 56 56 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 34 34 34 55 55 55 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 37 37 37 58 58 58 F F F 8 160-179 34 34 34 55 55 55 F F F 9 180-199 38 38 38 61 61 61 T T T 10 200-219 30 30 30 51 51 51 T T T 11 220-239 38 38 38 61 61 61 F F F 12 240-259 36 36 36 57 57 57 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 35 35 35 56 56 56 F F F 15 300-319 35 35 35 56 56 56 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 38 38 38 61 61 61 F F F 19 380-399 38 38 38 61 61 61 F F F 20 400-419 38 38 38 60 60 60 F F F 21 420-439 35 35 35 56 56 56 F F F 22 440-459 36 36 36 57 57 57 T T T 23 460-479 38 38 38 62 62 62 F F F 24 480-499 35 35 35 56 56 56 F F F 25 500-519 38 38 38 62 62 62 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 36 36 36 57 57 57 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 38 38 38 60 60 60 F F F Max gen: 38, Max flush: 62 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 37 37 37 58 58 58 F F F 3 060-079 35 35 35 56 56 56 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 34 34 34 55 55 55 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 37 37 37 58 58 58 F F F 8 160-179 34 34 34 55 55 55 F F F 9 180-199 39 39 39 63 63 63 F F F 10 200-219 39 39 39 63 63 63 F F F 11 220-239 38 38 38 61 61 61 F F F 12 240-259 36 36 36 57 57 57 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 35 35 35 56 56 56 F F F 15 300-319 35 35 35 56 56 56 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 38 38 38 61 61 61 F F F 19 380-399 38 38 38 61 61 61 F F F 20 400-419 38 38 38 60 60 60 F F F 21 420-439 35 35 35 56 56 56 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 35 35 35 56 56 56 F F F 25 500-519 38 38 38 62 62 62 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 36 36 36 57 57 57 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 38 38 38 60 60 60 F F F Max gen: 39, Max flush: 63 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.35400869Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.39818311Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.398805654Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.39884762Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.398864438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.398882126Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352} {"msg":"Crucible 5577175c-c16b-41f8-8331-1251d17bc8c6 has session id: dd536bcc-c210-44b4-a624-e2004543dfdc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.399401662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Upstairs opts: Upstairs UUID: 5577175c-c16b-41f8-8331-1251d17bc8c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.399441408Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Crucible stats registered with UUID: 5577175c-c16b-41f8-8331-1251d17bc8c6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.399463355Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:46.399482593Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400034835Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400065662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352} {"msg":"5577175c-c16b-41f8-8331-1251d17bc8c6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400116577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400143194Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"0","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400172141Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"0","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400190809Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400205407Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400218546Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"2","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400232794Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"2","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400245883Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.40026851Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400291138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:65036)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400318885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:46.400345813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:46.400374599Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:46.400404026Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400578498Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:53206)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.400623343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.401041669Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"2","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.401169816Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"0","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.401624488Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"2","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.401764193Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"0","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.402188049Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:46.402352212Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.401861857Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:38995)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.401987014Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.403105676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.403670047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[0]R flush_numbers[0..12]: [62, 63, 58, 56, 40, 55, 57, 58, 55, 63, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404281322Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[0]R generation[0..12]: [38, 39, 37, 35, 22, 34, 36, 37, 34, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.40430206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404317229Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[1]R flush_numbers[0..12]: [62, 63, 58, 56, 40, 55, 57, 58, 55, 63, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404330787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[1]R generation[0..12]: [38, 39, 37, 35, 22, 34, 36, 37, 34, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404344116Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404362654Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[2]R flush_numbers[0..12]: [62, 63, 58, 56, 40, 55, 57, 58, 55, 63, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404376253Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[2]R generation[0..12]: [38, 39, 37, 35, 22, 34, 36, 37, 34, 39, 39, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404389371Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.40440298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404415528Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404428247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404440926Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404453424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"":"downstairs","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404466343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404477962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"0","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404491091Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"1","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404512628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"client":"2","":"downstairs","session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404530636Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404542915Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"5577175c-c16b-41f8-8331-1251d17bc8c6 is now active with session: dd536bcc-c210-44b4-a624-e2004543dfdc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404555534Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404569832Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352,"session_id":"dd536bcc-c210-44b4-a624-e2004543dfdc"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.404581861Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352} {"msg":"Activated sub_volume 5577175c-c16b-41f8-8331-1251d17bc8c6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:16:56.40459267Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 395 len 40960 data: 3 4 3 3 3 4 5 5 4 5 03/30 Read block 80 len 40960 04/30 Read block 427 len 32768 05/30 Write block 372 len 32768 data: 3 3 3 3 4 4 4 4 06/30 Read block 210 len 20480 07/30 Read block 296 len 8192 08/30 Read block 586 len 8192 09/30 Write block 56 len 8192 data: 6 7 10/30 Write block 101 len 40960 data: 6 6 5 5 4 4 4 3 3 4 11/30 Write block 393 len 36864 data: 4 3 4 5 4 4 4 5 6 12/30 Write block 206 len 20480 data: 2 2 2 2 2 13/30 Read block 44 len 4096 14/30 Read block 159 len 4096 15/30 Read block 313 len 12288 16/30 Read block 577 len 32768 17/30 Read block 106 len 28672 18/30 Write block 162 len 36864 data: 4 4 4 3 4 4 4 3 3 19/30 Read block 40 len 12288 20/30 Write block 419 len 4096 data: 8 21/30 Read block 133 len 4096 22/30 Read block 140 len 32768 23/30 Read block 399 len 36864 24/30 Read block 169 len 32768 25/30 Read block 374 len 32768 26/30 Write block 374 len 32768 data: 4 4 5 5 5 5 4 4 27/30 Write block 311 len 36864 data: 5 5 4 5 4 3 3 3 3 28/30 Write block 291 len 8192 data: 4 7 29/30 Read block 538 len 12288 30/30 Read block 137 len 32768 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 10 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:16:56.426588199Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 37 37 37 58 58 58 T T T 3 060-079 35 35 35 56 56 56 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 34 34 34 55 55 55 T T T 6 120-139 36 36 36 57 57 57 F F F 7 140-159 37 37 37 58 58 58 F F F 8 160-179 34 34 34 55 55 55 T T T 9 180-199 39 39 39 63 63 63 F F F 10 200-219 39 39 39 63 63 63 T T T 11 220-239 38 38 38 61 61 61 F F F 12 240-259 36 36 36 57 57 57 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 35 35 35 56 56 56 T T T 15 300-319 35 35 35 56 56 56 T T T 16 320-339 36 36 36 57 57 57 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 38 38 38 61 61 61 T T T 19 380-399 38 38 38 61 61 61 T T T 20 400-419 38 38 38 60 60 60 T T T 21 420-439 35 35 35 56 56 56 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 35 35 35 56 56 56 F F F 25 500-519 38 38 38 62 62 62 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 36 36 36 57 57 57 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 38 38 38 60 60 60 F F F Max gen: 39, Max flush: 63 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 40 40 40 64 64 64 F F F 3 060-079 35 35 35 56 56 56 F F F 4 080-099 22 22 22 40 40 40 F F F 5 100-119 40 40 40 64 64 64 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 37 37 37 58 58 58 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 39 39 39 63 63 63 F F F 10 200-219 40 40 40 64 64 64 F F F 11 220-239 38 38 38 61 61 61 F F F 12 240-259 36 36 36 57 57 57 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 40 40 40 64 64 64 F F F 15 300-319 40 40 40 64 64 64 F F F 16 320-339 36 36 36 57 57 57 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 40 40 40 64 64 64 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 40 40 40 64 64 64 F F F 21 420-439 35 35 35 56 56 56 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 35 35 35 56 56 56 F F F 25 500-519 38 38 38 62 62 62 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 36 36 36 57 57 57 F F F 28 560-579 29 29 29 49 49 49 F F F 29 580-599 38 38 38 60 60 60 F F F Max gen: 40, Max flush: 64 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.68920748Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.735489298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.736110623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.736145379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.736159778Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.736173956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360} {"msg":"Crucible 32b487f1-7d76-4523-8228-10cda5a972ab has session id: 94c163b7-d49b-47ea-b1a9-50a8110a4644","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.736660115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Upstairs opts: Upstairs UUID: 32b487f1-7d76-4523-8228-10cda5a972ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.736698291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Crucible stats registered with UUID: 32b487f1-7d76-4523-8228-10cda5a972ab","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.736720339Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:00.736738717Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737231925Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737255613Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360} {"msg":"32b487f1-7d76-4523-8228-10cda5a972ab active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737332584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737349463Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"0","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737378249Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"0","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737397558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"1","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737411956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"1","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737424705Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"2","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737438333Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"2","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737450992Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.73746424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737482039Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"ds_connection connected from Ok(127.0.0.1:50326)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737534513Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"ds_connection connected from Ok(127.0.0.1:58296)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737573079Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"ds_connection connected from Ok(127.0.0.1:61680)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.737770148Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.738217931Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"1","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.738248978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"0","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.738909769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"0","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.738947585Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"1","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.73946433Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.739578309Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.73965508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"2","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740175956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"2","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[0]R flush_numbers[0..12]: [62, 63, 64, 56, 40, 64, 57, 58, 64, 63, 64, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740663435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[0]R generation[0..12]: [38, 39, 40, 35, 22, 40, 36, 37, 40, 39, 40, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740687592Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.74070312Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[1]R flush_numbers[0..12]: [62, 63, 64, 56, 40, 64, 57, 58, 64, 63, 64, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740716989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[1]R generation[0..12]: [38, 39, 40, 35, 22, 40, 36, 37, 40, 39, 40, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740730357Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740743906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[2]R flush_numbers[0..12]: [62, 63, 64, 56, 40, 64, 57, 58, 64, 63, 64, 61]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740756895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[2]R generation[0..12]: [38, 39, 40, 35, 22, 40, 36, 37, 40, 39, 40, 38]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740769883Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740783452Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Max found gen is 41","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.7408033Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid"Wait for a query_work_queue command to finish before sending IO :1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Generation requested: 42 >= found:41","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740875222Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740902179Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740927987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"":"downstairs","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740953624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.740977642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"0","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.741004389Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"1","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.741029336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"client":"2","":"downstairs","session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.741055003Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.741078491Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"32b487f1-7d76-4523-8228-10cda5a972ab is now active with session: 94c163b7-d49b-47ea-b1a9-50a8110a4644","v":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.741109638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.741134795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360,"session_id":"94c163b7-d49b-47ea-b1a9-50a8110a4644"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.741152803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360} {"msg":"Activated sub_volume 32b487f1-7d76-4523-8228-10cda5a972ab","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:00.741165052Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 68 len 40960 data: 4 4 4 4 4 3 3 2 2 4 02/30 Read block 138 len 24576 03/30 Write block 219 len 36864 data: 3 3 3 4 5 4 3 3 3 04/30 Read block 283 len 4096 05/30 Read block 381 len 12288 06/30 Read block 103 len 32768 07/30 Write block 558 len 4096 data: 4 08/30 Write block 481 len 40960 data: 3 3 2 3 3 3 3 3 3 3 09/30 Read block 572 len 36864 10/30 Read block 271 len 4096 11/30 Read block 130 len 32768 12/30 Read block 248 len 24576 13/30 Write block 500 len 28672 data: 4 4 5 5 4 5 4 14/30 Write block 547 len 28672 data: 3 3 4 4 3 3 3 15/30 Write block 292 len 8192 data: 8 8 16/30 Write block 94 len 28672 data: 3 4 4 4 4 5 6 17/30 Read block 34 len 32768 18/30 Flush 19/30 Write block 405 len 16384 data: 5 6 6 6 20/30 Read block 126 len 4096 21/30 Write block 568 len 32768 data: 2 2 2 2 4 4 3 3 22/30 Write block 184 len 12288 data: 6 6 7 23/30 Read block 289 len 32768 24/30 Read block 135 len 12288 25/30 Read block 30 len 4096 26/30 Write block 341 len 8192 data: 5 4 27/30 Read block 170 len 4096 28/30 Write block 313 len 40960 data: 5 6 5 4 4 4 4 2 4 4 29/30 Write block 209 len 12288 data: 3 3 3 30/30 Write block 283 len 32768 data: 7 8 8 7 7 7 6 4 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:00.761031485Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 40 40 40 64 64 64 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 42 42 42 65 65 65 F F F 5 100-119 42 42 42 65 65 65 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 37 37 37 58 58 58 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 39 39 39 63 63 63 T T T 10 200-219 42 42 42 65 65 65 T T T 11 220-239 42 42 42 65 65 65 F F F 12 240-259 36 36 36 57 57 57 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 42 42 42 65 65 65 T T T 15 300-319 40 40 40 64 64 64 T T T 16 320-339 36 36 36 57 57 57 T T T 17 340-359 31 31 31 52 52 52 T T T 18 360-379 40 40 40 64 64 64 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 40 40 40 64 64 64 T T T 21 420-439 35 35 35 56 56 56 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 42 42 42 65 65 65 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 42 42 42 65 65 65 F F F 28 560-579 29 29 29 49 49 49 T T T 29 580-599 38 38 38 60 60 60 F F F Max gen: 42, Max flush: 65 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 40 40 40 64 64 64 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 42 42 42 65 65 65 F F F 5 100-119 42 42 42 65 65 65 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 37 37 37 58 58 58 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 43 43 43 66 66 66 F F F 10 200-219 43 43 43 66 66 66 F F F 11 220-239 42 42 42 65 65 65 F F F 12 240-259 36 36 36 57 57 57 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 43 43 43 66 66 66 F F F 15 300-319 43 43 43 66 66 66 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 43 43 43 66 66 66 F F F 18 360-379 40 40 40 64 64 64 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 43 43 43 66 66 66 F F F 21 420-439 35 35 35 56 56 56 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 42 42 42 65 65 65 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 42 42 42 65 65 65 F F F 28 560-579 43 43 43 66 66 66 F F F 29 580-599 38 38 38 60 60 60 F F F Max gen: 43, Max flush: 66 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.023423311Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.067049658Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.067684271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.067726947Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.067743885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.067777011Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368} {"msg":"Crucible 9c7002a9-833b-4f66-bf3b-96e29fcb309e has session id: 29c478cf-27f1-49fa-a37d-e9e2e8011ba3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.068418754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Upstairs opts: Upstairs UUID: 9c7002a9-833b-4f66-bf3b-96e29fcb309e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.068447091Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Crucible stats registered with UUID: 9c7002a9-833b-4f66-bf3b-96e29fcb309e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.068465799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:15.068479068Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.068954698Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.068989084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368} {"msg":"9c7002a9-833b-4f66-bf3b-96e29fcb309e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069059276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069078074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"0","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069105922Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"0","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.06912013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"1","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069134229Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"1","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069147227Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"2","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069161296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"2","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069174244Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069192993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.06921896Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38940)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069251047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"ds_connection connected from Ok(127.0.0.1:39668)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069328868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"ds_connection connected from Ok(127.0.0.1:51596)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069360975Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069926676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"1","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.069954093Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"0","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.070564688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"0","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.070682836Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"1","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.071112801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.071285333Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.071747114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"2","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072248812Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"2","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[0]R flush_numbers[0..12]: [62, 63, 64, 65, 65, 65, 57, 58, 64, 66, 66, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.07274108Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[0]R generation[0..12]: [38, 39, 40, 42, 42, 42, 36, 37, 40, 43, 43, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072760258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072775596Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[1]R flush_numbers[0..12]: [62, 63, 64, 65, 65, 65, 57, 58, 64, 66, 66, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072790155Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[1]R generation[0..12]: [38, 39, 40, 42, 42, 42, 36, 37, 40, 43, 43, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072817642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072845019Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[2]R flush_numbers[0..12]: [62, 63, 64, 65, 65, 65, 57, 58, 64, 66, 66, 65]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072863837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[2]R generation[0..12]: [38, 39, 40, 42, 42, 42, 36, 37, 40, 43, 43, 42]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072878905Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072892934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072905423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072918101Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.07293053Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072943268Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"":"downstairs","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072956427Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072974595Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"0","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.072987814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"1","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.073000573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"client":"2","":"downstairs","session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.073013521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.07302549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"9c7002a9-833b-4f66-bf3b-96e29fcb309e is now active with session: 29c478cf-27f1-49fa-a37d-e9e2e8011ba3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.073040378Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.073052117Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368,"session_id":"29c478cf-27f1-49fa-a37d-e9e2e8011ba3"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.073064186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368} {"msg":"Activated sub_volume 9c7002a9-833b-4f66-bf3b-96e29fcb309e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:15.073075125Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 301 len 36864 data: 5 3 3 3 2 3 5 5 5 02/30 Write block 289 len 24576 data: 7 5 5 9 9 8 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Write block 481 len 16384 data: 4 4 3 4 07/30 Read block 84 len 28672 08/30 Read block 542 len 40960 09/30 Read block 278 len 16384 10/30 Read block 493 len 40960 11/30 Read block 522 len 20480 12/30 Flush 13/30 Write block 423 len 16384 data: 2 2 3 4 14/30 Read block 41 len 36864 15/30 Write block 94 len 12288 data: 4 5 5 16/30 Read block 497 len 28672 17/30 Read block 276 len 8192 18/30 Write block 596 len 4096 data: 2 19/30 Read block 247 len 40960 20/30 Write block 140 len 20480 data: 4 4 4 5 5 21/30 Read block 224 len 32768 22/30 Read block 188 len 20480 23/30 Flush 24/30 Write block 238 len 40960 data: 3 5 5 5 4 4 4 4 3 4 25/30 Read block 467 len 36864 26/30 Read block 27 len 20480 27/30 Read block 2 len 20480 28/30 Write block 365 len 36864 data: 6 5 5 3 3 3 4 4 4 29/30 Read block 574 len 8192 30/30 Read block 539 len 12288 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:15.094106995Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 40 40 40 64 64 64 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 44 44 44 71 71 71 F F F 5 100-119 42 42 42 65 65 65 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 43 43 43 66 66 66 F F F 10 200-219 43 43 43 66 66 66 F F F 11 220-239 42 42 42 65 65 65 T T T 12 240-259 36 36 36 57 57 57 T T T 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 F F F 15 300-319 44 44 44 67 67 67 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 43 43 43 66 66 66 F F F 18 360-379 40 40 40 64 64 64 T T T 19 380-399 40 40 40 64 64 64 F F F 20 400-419 43 43 43 66 66 66 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 44 44 44 70 70 70 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 42 42 42 65 65 65 F F F 28 560-579 43 43 43 66 66 66 F F F 29 580-599 44 44 44 71 71 71 F F F Max gen: 44, Max flush: 71 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 40 40 40 64 64 64 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 44 44 44 71 71 71 F F F 5 100-119 42 42 42 65 65 65 F F F 6 120-139 36 36 36 57 57 57 F F F 7 140-159 44 44 44 71 71 71 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 43 43 43 66 66 66 F F F 10 200-219 43 43 43 66 66 66 F F F 11 220-239 45 45 45 72 72 72 F F F 12 240-259 45 45 45 72 72 72 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 F F F 15 300-319 44 44 44 67 67 67 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 43 43 43 66 66 66 F F F 18 360-379 45 45 45 72 72 72 F F F 19 380-399 40 40 40 64 64 64 F F F 20 400-419 43 43 43 66 66 66 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 44 44 44 70 70 70 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 35 35 35 56 56 56 F F F 27 540-559 42 42 42 65 65 65 F F F 28 560-579 43 43 43 66 66 66 F F F 29 580-599 44 44 44 71 71 71 F F F Max gen: 45, Max flush: 72 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.322712573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.36643001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.367082562Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.367114029Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.367133087Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.367147985Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376} {"msg":"Crucible 97c46283-4196-4ca2-a63b-72d37ea3ed85 has session id: 39ad738c-d53e-4901-82d2-28bd97f77ce8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.367679659Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Upstairs opts: Upstairs UUID: 97c46283-4196-4ca2-a63b-72d37ea3ed85, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.367716505Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Crucible stats registered with UUID: 97c46283-4196-4ca2-a63b-72d37ea3ed85","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.367737673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:29.367756131Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368223212Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368247019Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376} {"msg":"97c46283-4196-4ca2-a63b-72d37ea3ed85 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368324471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368342109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"0","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368369897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"0","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368390334Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"1","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368416342Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"1","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368441649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368474835Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368502113Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.3685326Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368560566Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:29.368590483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:29.36862023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38519)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368649687Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:29.368731559Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"ds_connection connected from Ok(127.0.0.1:50482)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368803621Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.368828808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.369293389Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"1","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.369330676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"0","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.369977828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"1","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.370006405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"0","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.370630969Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:29.370660446Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.370442642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56699)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.370527573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.371654224Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.372380998Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[0]R flush_numbers[0..12]: [62, 63, 64, 65, 71, 65, 57, 71, 64, 66, 66, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373015801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[0]R generation[0..12]: [38, 39, 40, 42, 44, 42, 36, 44, 40, 43, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373052538Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373078975Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[1]R flush_numbers[0..12]: [62, 63, 64, 65, 71, 65, 57, 71, 64, 66, 66, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373104282Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[1]R generation[0..12]: [38, 39, 40, 42, 44, 42, 36, 44, 40, 43, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.37312714Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373151677Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[2]R flush_numbers[0..12]: [62, 63, 64, 65, 71, 65, 57, 71, 64, 66, 66, 72]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373177624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[2]R generation[0..12]: [38, 39, 40, 42, 44, 42, 36, 44, 40, 43, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373206801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373242767Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:17:39.373269505Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373299712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373326219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373342327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"":"downstairs","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373356106Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373368334Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"0","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373381623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"1","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373394271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"client":"2","":"downstairs","session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.37340702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373419109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"97c46283-4196-4ca2-a63b-72d37ea3ed85 is now active with session: 39ad738c-d53e-4901-82d2-28bd97f77ce8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373431508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373443246Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376,"session_id":"39ad738c-d53e-4901-82d2-28bd97f77ce8"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373455265Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376} {"msg":"Activated sub_volume 97c46283-4196-4ca2-a63b-72d37ea3ed85","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:39.373466254Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 132 len 36864 data: 3 4 4 3 3 3 4 4 5 02/30 Write block 184 len 4096 data: 7 03/30 Write block 584 len 4096 data: 7 04/30 Read block 92 len 40960 05/30 Read block 39 len 24576 06/30 Write block 346 len 12288 data: 2 4 5 07/30 Write block 99 len 20480 data: 6 7 7 7 6 08/30 Read block 516 len 24576 09/30 Write block 481 len 40960 data: 5 5 4 5 4 4 4 4 4 4 10/30 Read block 281 len 20480 11/30 Read block 424 len 40960 12/30 Write block 366 len 36864 data: 6 6 4 4 4 5 5 5 5 13/30 Read block 239 len 16384 14/30 Write block 534 len 24576 data: 6 6 6 7 7 6 15/30 Read block 509 len 36864 16/30 Read block 479 len 16384 17/30 Read block 386 len 40960 18/30 Read block 402 len 32768 19/30 Read block 31 len 40960 20/30 Write block 111 len 4096 data: 4 21/30 Read block 540 len 40960 22/30 Read block 222 len 24576 23/30 Read block 129 len 28672 24/30 Write block 377 len 32768 data: 6 6 6 5 5 4 5 6 25/30 Flush 26/30 Write block 217 len 40960 data: 4 2 4 4 4 5 6 5 4 4 27/30 Write block 349 len 36864 data: 5 4 4 4 4 4 3 3 2 28/30 Read block 168 len 8192 29/30 Read block 22 len 16384 30/30 Write block 122 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:39.395095372Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 40 40 40 64 64 64 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 46 46 46 73 73 73 F F F 5 100-119 46 46 46 73 73 73 F F F 6 120-139 46 46 46 73 73 73 T T T 7 140-159 46 46 46 73 73 73 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 43 43 43 66 66 66 T T T 11 220-239 45 45 45 72 72 72 T T T 12 240-259 45 45 45 72 72 72 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 F F F 15 300-319 44 44 44 67 67 67 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 46 46 46 73 73 73 T T T 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 43 43 43 66 66 66 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 46 46 46 73 73 73 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 42 42 42 65 65 65 F F F 28 560-579 43 43 43 66 66 66 F F F 29 580-599 46 46 46 73 73 73 F F F Max gen: 46, Max flush: 73 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 39 39 39 63 63 63 F F F 2 040-059 40 40 40 64 64 64 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 46 46 46 73 73 73 F F F 5 100-119 46 46 46 73 73 73 F F F 6 120-139 47 47 47 74 74 74 F F F 7 140-159 46 46 46 73 73 73 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 47 47 47 74 74 74 F F F 11 220-239 47 47 47 74 74 74 F F F 12 240-259 45 45 45 72 72 72 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 F F F 15 300-319 44 44 44 67 67 67 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 47 47 47 74 74 74 F F F 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 43 43 43 66 66 66 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 39 39 39 63 63 63 F F F 23 460-479 38 38 38 62 62 62 F F F 24 480-499 46 46 46 73 73 73 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 42 42 42 65 65 65 F F F 28 560-579 43 43 43 66 66 66 F F F 29 580-599 46 46 46 73 73 73 F F F Max gen: 47, Max flush: 74 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.62580285Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:53.66787848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.668234802Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.698988692Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.699580109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.699612786Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.699626214Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.699640913Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384} {"msg":"Crucible e0b42a31-8b3c-43e7-a351-1dae3a8ee736 has session id: e009be30-2786-4023-811f-5b24d17c2329","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700244829Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Upstairs opts: Upstairs UUID: e0b42a31-8b3c-43e7-a351-1dae3a8ee736, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700289575Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Crucible stats registered with UUID: e0b42a31-8b3c-43e7-a351-1dae3a8ee736","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700312073Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:53.700327301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700749567Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700781183Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384} {"msg":"e0b42a31-8b3c-43e7-a351-1dae3a8ee736 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700835487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700862095Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"0","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700891012Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"0","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.70090588Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"1","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700928868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"1","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700943346Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"2","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.700987951Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"2","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.701033877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.701065073Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.701087991Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"ds_connection connected from Ok(127.0.0.1:43762)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.70110261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"ds_connection connected from Ok(127.0.0.1:41648)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.701122677Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"ds_connection connected from Ok(127.0.0.1:38052)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.701231816Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.701786998Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"2","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.701822884Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"1","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.701893036Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"0","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.702376286Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"1","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.702488724Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"2","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.702518031Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"0","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.702975293Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703085231Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[0]R flush_numbers[0..12]: [62, 63, 64, 65, 73, 73, 74, 73, 64, 73, 74, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703123077Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[0]R generation[0..12]: [38, 39, 40, 42, 46, 46, 47, 46, 40, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703138725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703152864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[1]R flush_numbers[0..12]: [62, 63, 64, 65, 73, 73, 74, 73, 64, 73, 74, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703170672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[1]R generation[0..12]: [38, 39, 40, 42, 46, 46, 47, 46, 40, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703184901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703198789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[2]R flush_numbers[0..12]: [62, 63, 64, 65, 73, 73, 74, 73, 64, 73, 74, 74]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703212128Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[2]R generation[0..12]: [38, 39, 40, 42, 46, 46, 47, 46, 40, 46, 47, 47]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703225376Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703238885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703252084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703264802Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703277501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.70329138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"":"downstairs","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:17:53.703310747Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703329296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"0","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703342414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"1","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703355133Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"client":"2","":"downstairs","session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703370041Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.70338226Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"e0b42a31-8b3c-43e7-a351-1dae3a8ee736 is now active with session: e009be30-2786-4023-811f-5b24d17c2329","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703402378Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703418586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384,"session_id":"e009be30-2786-4023-811f-5b24d17c2329"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703430805Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384} {"msg":"Activated sub_volume e0b42a31-8b3c-43e7-a351-1dae3a8ee736","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:53.703441674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 466 len 36864 data: 3 4 4 4 6 6 6 6 5 03/30 Read block 96 len 40960 04/30 Write block 401 len 12288 data: 7 6 5 05/30 Write block 239 len 28672 data: 6 6 6 5 5 5 5 06/30 Flush 07/30 Write block 556 len 4096 data: 3 08/30 Write block 569 len 20480 data: 3 3 3 5 5 09/30 Write block 38 len 40960 data: 6 3 4 5 5 4 4 4 4 3 10/30 Write block 211 len 32768 data: 4 3 4 4 5 5 5 3 11/30 Flush 12/30 Write block 142 len 40960 data: 5 6 6 5 6 6 6 4 4 4 13/30 Write block 473 len 8192 data: 7 6 14/30 Flush 15/30 Write block 203 len 24576 data: 4 2 2 3 3 3 16/30 Read block 276 len 16384 17/30 Read block 409 len 8192 18/30 Read block 542 len 40960 19/30 Flush 20/30 Write block 492 len 12288 data: 4 4 4 21/30 Write block 212 len 28672 data: 4 5 5 6 6 6 4 22/30 Write block 546 len 16384 data: 3 4 4 5 23/30 Read block 250 len 36864 24/30 Read block 441 len 20480 25/30 Write block 543 len 8192 data: 3 3 26/30 Write block 464 len 12288 data: 5 3 4 27/30 Write block 446 len 36864 data: 6 4 5 5 4 6 5 4 5 28/30 Read block 181 len 24576 29/30 Read block 147 len 16384 30/30 Read block 202 len 16384 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 3 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 4 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 10 11 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:53.723731922Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 48 48 48 77 77 77 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 46 46 46 73 73 73 F F F 5 100-119 46 46 46 73 73 73 F F F 6 120-139 47 47 47 74 74 74 F F F 7 140-159 48 48 48 78 78 78 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 48 48 48 79 79 79 T T T 11 220-239 48 48 48 76 76 76 F F F 12 240-259 48 48 48 76 76 76 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 F F F 15 300-319 44 44 44 67 67 67 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 47 47 47 74 74 74 F F F 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 48 48 48 76 76 76 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 39 39 39 63 63 63 T T T 23 460-479 48 48 48 78 78 78 T T T 24 480-499 46 46 46 73 73 73 T T T 25 500-519 42 42 42 65 65 65 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 48 48 48 77 77 77 T T T 28 560-579 48 48 48 77 77 77 F F F 29 580-599 46 46 46 73 73 73 F F F Max gen: 48, Max flush: 79 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 48 48 48 77 77 77 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 46 46 46 73 73 73 F F F 5 100-119 46 46 46 73 73 73 F F F 6 120-139 47 47 47 74 74 74 F F F 7 140-159 48 48 48 78 78 78 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 49 49 49 80 80 80 F F F 11 220-239 48 48 48 76 76 76 F F F 12 240-259 48 48 48 76 76 76 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 F F F 15 300-319 44 44 44 67 67 67 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 47 47 47 74 74 74 F F F 18 360-379 46 46 46 73 73 73 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 48 48 48 76 76 76 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 49 49 49 80 80 80 F F F 23 460-479 49 49 49 80 80 80 F F F 24 480-499 49 49 49 80 80 80 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 49 49 49 80 80 80 F F F 28 560-579 48 48 48 77 77 77 F F F 29 580-599 46 46 46 73 73 73 F F F Max gen: 49, Max flush: 80 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:57.989862405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.033955773Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.034688856Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.034731982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.03474962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.034767578Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392} {"msg":"Crucible e085ef09-6f70-4a31-85d5-95f4824e738c has session id: 69d66cb4-9459-44ad-98a2-f57cd6eb04b0","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.035305761Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Upstairs opts: Upstairs UUID: e085ef09-6f70-4a31-85d5-95f4824e738c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.035337548Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Crucible stats registered with UUID: e085ef09-6f70-4a31-85d5-95f4824e738c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.035352957Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:58.035365605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.035875522Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.035922837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392} {"msg":"e085ef09-6f70-4a31-85d5-95f4824e738c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.035994349Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036019156Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"0","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036055463Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"0","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036086269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"1","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036153052Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"1","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036184719Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036209966Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036233634Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036259541Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036285519Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44751)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036312536Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50104)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036340023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:58.036385638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:58.036417854Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:17:58.036439462Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.036573208Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.037048838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"1","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.037080125Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"0","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.037689621Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"1","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.037717068Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"0","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.038354651Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:17:58.038391797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.037504895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63535)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.037660128Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.038753923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.039324934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[0]R flush_numbers[0..12]: [62, 77, 77, 65, 73, 73, 74, 78, 64, 73, 80, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.039985784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[0]R generation[0..12]: [38, 48, 48, 42, 46, 46, 47, 48, 40, 46, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040036929Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040059016Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[1]R flush_numbers[0..12]: [62, 77, 77, 65, 73, 73, 74, 78, 64, 73, 80, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040076475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[1]R generation[0..12]: [38, 48, 48, 42, 46, 46, 47, 48, 40, 46, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040093423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040110451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[2]R flush_numbers[0..12]: [62, 77, 77, 65, 73, 73, 74, 78, 64, 73, 80, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040132669Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[2]R generation[0..12]: [38, 48, 48, 42, 46, 46, 47, 48, 40, 46, 49, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040149667Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040166555Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040182324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040198512Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.04021406Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid"Wait for a query_work_queue command to finish before sending IO :1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040238558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"":"downstairs","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040254616Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040269224Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"0","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040285773Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"1","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040302191Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"client":"2","":"downstairs","session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040318679Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040333548Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"e085ef09-6f70-4a31-85d5-95f4824e738c is now active with session: 69d66cb4-9459-44ad-98a2-f57cd6eb04b0","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040349296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040364014Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392,"session_id":"69d66cb4-9459-44ad-98a2-f57cd6eb04b0"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040381583Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392} {"msg":"Activated sub_volume e085ef09-6f70-4a31-85d5-95f4824e738c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:08.040395451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 8192 data: 4 4 02/30 Write block 457 len 40960 data: 6 4 6 6 6 6 5 6 4 5 03/30 Write block 210 len 16384 data: 4 5 5 6 04/30 Write block 408 len 28672 data: 7 5 7 6 5 4 4 05/30 Read block 446 len 24576 06/30 Read block 574 len 4096 07/30 Read block 440 len 4096 08/30 Read block 30 len 12288 09/30 Read block 394 len 20480 10/30 Read block 260 len 4096 11/30 Flush 12/30 Flush 13/30 Write block 409 len 12288 data: 6 8 7 14/30 Read block 424 len 8192 15/30 Write block 99 len 28672 data: 7 8 8 8 7 6 5 16/30 Read block 166 len 24576 17/30 Read block 568 len 16384 18/30 Read block 337 len 24576 19/30 Write block 593 len 20480 data: 2 2 3 3 2 20/30 Write block 202 len 36864 data: 4 5 3 3 4 4 4 4 5 21/30 Write block 581 len 40960 data: 4 4 5 8 7 7 7 6 5 3 22/30 Read block 268 len 16384 23/30 Read block 104 len 12288 24/30 Read block 570 len 4096 25/30 Read block 350 len 8192 26/30 Write block 142 len 16384 data: 6 7 7 6 27/30 Read block 221 len 36864 28/30 Read block 39 len 12288 29/30 Read block 447 len 20480 30/30 Write block 359 len 24576 data: 2 2 2 4 5 4 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:08.060546544Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 48 48 48 77 77 77 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 46 46 46 73 73 73 T T T 5 100-119 50 50 50 81 81 81 T T T 6 120-139 47 47 47 74 74 74 F F F 7 140-159 48 48 48 78 78 78 T T T 8 160-179 40 40 40 64 64 64 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 50 50 50 81 81 81 T T T 11 220-239 48 48 48 76 76 76 F F F 12 240-259 48 48 48 76 76 76 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 F F F 15 300-319 44 44 44 67 67 67 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 47 47 47 74 74 74 T T T 18 360-379 46 46 46 73 73 73 T T T 19 380-399 46 46 46 73 73 73 F F F 20 400-419 50 50 50 81 81 81 T T T 21 420-439 44 44 44 71 71 71 F F F 22 440-459 50 50 50 81 81 81 F F F 23 460-479 50 50 50 81 81 81 F F F 24 480-499 49 49 49 80 80 80 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 49 49 49 80 80 80 F F F 28 560-579 48 48 48 77 77 77 F F F 29 580-599 46 46 46 73 73 73 T T T Max gen: 50, Max flush: 81 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 F F F 1 020-039 48 48 48 77 77 77 F F F 2 040-059 48 48 48 77 77 77 F F F 3 060-079 42 42 42 65 65 65 F F F 4 080-099 51 51 51 82 82 82 F F F 5 100-119 51 51 51 82 82 82 F F F 6 120-139 47 47 47 74 74 74 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 51 51 51 82 82 82 F F F 11 220-239 48 48 48 76 76 76 F F F 12 240-259 48 48 48 76 76 76 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 F F F 15 300-319 44 44 44 67 67 67 F F F 16 320-339 43 43 43 66 66 66 F F F 17 340-359 51 51 51 82 82 82 F F F 18 360-379 51 51 51 82 82 82 F F F 19 380-399 46 46 46 73 73 73 F F F 20 400-419 51 51 51 82 82 82 F F F 21 420-439 44 44 44 71 71 71 F F F 22 440-459 50 50 50 81 81 81 F F F 23 460-479 50 50 50 81 81 81 F F F 24 480-499 49 49 49 80 80 80 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 49 49 49 80 80 80 F F F 28 560-579 48 48 48 77 77 77 F F F 29 580-599 51 51 51 82 82 82 F F F Max gen: 51, Max flush: 82 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.312877266Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.356028383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.356619781Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.356654477Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.356673825Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.356689114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400} {"msg":"Crucible 796e5bf8-3f03-404c-b6e5-70e47c795a3e has session id: 253e56bc-d93e-4f6b-b438-d037cd64fafe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357220998Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Upstairs opts: Upstairs UUID: 796e5bf8-3f03-404c-b6e5-70e47c795a3e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357260854Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Crucible stats registered with UUID: 796e5bf8-3f03-404c-b6e5-70e47c795a3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357285801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:12.357308619Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357764091Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357782889Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400} {"msg":"796e5bf8-3f03-404c-b6e5-70e47c795a3e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357858591Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357875759Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"0","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357904166Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"0","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357918215Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"1","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357932223Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"1","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357949081Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.35796307Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357975888Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.357988977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.358002126Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:12.358098046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"ds_connection connected from Ok(127.0.0.1:50005)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.358132872Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:12.358162139Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"ds_connection connected from Ok(127.0.0.1:48001)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.358188556Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:12.358217343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.358314953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.358733799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"0","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.358772365Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"1","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.359446574Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"0","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.359480261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"1","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.360021234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:12.360045601Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.359190263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"ds_connection connected from Ok(127.0.0.1:49029)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.359340037Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.360423043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361055307Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[0]R flush_numbers[0..12]: [62, 77, 77, 65, 82, 82, 74, 82, 64, 73, 82, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361681531Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[0]R generation[0..12]: [38, 48, 48, 42, 51, 51, 47, 51, 40, 46, 51, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361701859Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361716787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[1]R flush_numbers[0..12]: [62, 77, 77, 65, 82, 82, 74, 82, 64, 73, 82, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361730466Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[1]R generation[0..12]: [38, 48, 48, 42, 51, 51, 47, 51, 40, 46, 51, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361743744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361758433Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[2]R flush_numbers[0..12]: [62, 77, 77, 65, 82, 82, 74, 82, 64, 73, 82, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361771941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[2]R generation[0..12]: [38, 48, 48, 42, 51, 51, 47, 51, 40, 46, 51, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.36178547Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361805128Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361818067Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361830855Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361843234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361855483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"":"downstairs","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361868242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.36187986Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"0","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361892729Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"1","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361905478Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"client":"2","":"downstairs","session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361918626Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T15:18:22.361930795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"796e5bf8-3f03-404c-b6e5-70e47c795a3e is now active with session: 253e56bc-d93e-4f6b-b438-d037cd64fafe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361951233Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361962752Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400,"session_id":"253e56bc-d93e-4f6b-b438-d037cd64fafe"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.36197469Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400} {"msg":"Activated sub_volume 796e5bf8-3f03-404c-b6e5-70e47c795a3e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:22.361985729Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 32768 data: 3 4 4 3 3 2 3 3 02/30 Write block 485 len 16384 data: 5 5 5 5 03/30 Write block 361 len 40960 data: 3 5 6 5 7 7 7 5 5 5 04/30 Write block 95 len 20480 data: 6 6 5 5 8 05/30 Write block 128 len 36864 data: 4 4 3 3 4 5 5 4 4 06/30 Read block 498 len 40960 07/30 Write block 87 len 28672 data: 3 3 2 2 2 2 3 08/30 Write block 544 len 20480 data: 4 3 4 5 5 09/30 Write block 314 len 4096 data: 7 10/30 Read block 574 len 40960 11/30 Write block 429 len 40960 data: 6 8 8 7 7 7 7 4 4 5 12/30 Read block 427 len 40960 13/30 Write block 208 len 4096 data: 5 14/30 Read block 273 len 32768 15/30 Read block 332 len 12288 16/30 Read block 198 len 24576 17/30 Write block 319 len 24576 data: 5 3 5 5 4 6 18/30 Read block 417 len 20480 19/30 Write block 209 len 40960 data: 5 6 6 6 7 6 7 7 7 5 20/30 Read block 323 len 4096 21/30 Write block 391 len 36864 data: 4 4 5 4 5 6 5 5 5 22/30 Write block 74 len 16384 data: 4 3 3 5 23/30 Read block 15 len 16384 24/30 Read block 544 len 8192 25/30 Read block 524 len 12288 26/30 Write block 280 len 36864 data: 4 4 6 8 9 9 8 8 8 27/30 Write block 41 len 40960 data: 6 6 5 5 5 5 4 3 4 4 28/30 Read block 429 len 4096 29/30 Write block 560 len 28672 data: 3 3 3 2 2 2 3 30/30 Read block 224 len 12288 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 6 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 30 28 28 86 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:22.382682015Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 62 62 62 T T T 1 020-039 48 48 48 77 77 77 T T T 2 040-059 48 48 48 77 77 77 T T T 3 060-079 42 42 42 65 65 65 T T T 4 080-099 51 51 51 82 82 82 T T T 5 100-119 51 51 51 82 82 82 F F F 6 120-139 47 47 47 74 74 74 T T T 7 140-159 51 51 51 82 82 82 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 51 51 51 82 82 82 T T T 11 220-239 48 48 48 76 76 76 F F F 12 240-259 48 48 48 76 76 76 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 44 44 44 67 67 67 T T T 15 300-319 44 44 44 67 67 67 T T T 16 320-339 43 43 43 66 66 66 T T T 17 340-359 51 51 51 82 82 82 F F F 18 360-379 51 51 51 82 82 82 T T T 19 380-399 46 46 46 73 73 73 T T T 20 400-419 51 51 51 82 82 82 F F F 21 420-439 44 44 44 71 71 71 T T T 22 440-459 50 50 50 81 81 81 F F F 23 460-479 50 50 50 81 81 81 F F F 24 480-499 49 49 49 80 80 80 T T T 25 500-519 42 42 42 65 65 65 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 49 49 49 80 80 80 T T T 28 560-579 48 48 48 77 77 77 T T T 29 580-599 51 51 51 82 82 82 F F F Max gen: 51, Max flush: 82 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 83 83 83 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 52 52 52 83 83 83 F F F 3 060-079 52 52 52 83 83 83 F F F 4 080-099 52 52 52 83 83 83 F F F 5 100-119 51 51 51 82 82 82 F F F 6 120-139 52 52 52 83 83 83 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 40 40 40 64 64 64 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 52 52 52 83 83 83 F F F 11 220-239 48 48 48 76 76 76 F F F 12 240-259 48 48 48 76 76 76 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 52 52 52 83 83 83 F F F 15 300-319 52 52 52 83 83 83 F F F 16 320-339 52 52 52 83 83 83 F F F 17 340-359 51 51 51 82 82 82 F F F 18 360-379 52 52 52 83 83 83 F F F 19 380-399 52 52 52 83 83 83 F F F 20 400-419 51 51 51 82 82 82 F F F 21 420-439 52 52 52 83 83 83 F F F 22 440-459 50 50 50 81 81 81 F F F 23 460-479 50 50 50 81 81 81 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 42 42 42 65 65 65 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 52 52 52 83 83 83 F F F 28 560-579 52 52 52 83 83 83 F F F 29 580-599 51 51 51 82 82 82 F F F Max gen: 52, Max flush: 83 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.694253728Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.738610438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.739292447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.739333642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.73935079Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.739368619Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408} {"msg":"Crucible 77eb6b28-8fa1-46f0-96cb-4d513d302c56 has session id: 4092f6f4-a6bb-4f94-950f-29c06dc41d9c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.739997042Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Upstairs opts: Upstairs UUID: 77eb6b28-8fa1-46f0-96cb-4d513d302c56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740031159Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Crucible stats registered with UUID: 77eb6b28-8fa1-46f0-96cb-4d513d302c56","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740048617Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:36.740063946Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740599399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740631596Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408} {"msg":"77eb6b28-8fa1-46f0-96cb-4d513d302c56 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740723516Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740757013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"0","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740798228Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"0","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740818056Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"1","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740843753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"1","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740905557Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"2","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740930295Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"2","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740958612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.74097313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.740988029Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47663)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.741012926Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55742)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.741037693Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:45941)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.7410649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.741714032Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"0","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.741741589Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"2","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.742294321Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"1","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.742430687Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"2","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.742464803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"0","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.742843914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"1","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743077059Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743111775Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[0]R flush_numbers[0..12]: [83, 83, 83, 83, 83, 82, 83, 82, 64, 73, 83, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743471388Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[0]R generation[0..12]: [52, 52, 52, 52, 52, 51, 52, 51, 40, 46, 52, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743492795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743508314Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[1]R flush_numbers[0..12]: [83, 83, 83, 83, 83, 82, 83, 82, 64, 73, 83, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743522102Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[1]R generation[0..12]: [52, 52, 52, 52, 52, 51, 52, 51, 40, 46, 52, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743535371Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743549259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[2]R flush_numbers[0..12]: [83, 83, 83, 83, 83, 82, 83, 82, 64, 73, 83, 76]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743562448Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[2]R generation[0..12]: [52, 52, 52, 52, 52, 51, 52, 51, 40, 46, 52, 48]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743575717Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743589275Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743602324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743615362Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743627731Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743644689Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743666477Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743678786Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"0","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743692304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"1","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743704993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"client":"2","":"downstairs","session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743717632Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.74372953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"77eb6b28-8fa1-46f0-96cb-4d513d302c56 is now active with session: 4092f6f4-a6bb-4f94-950f-29c06dc41d9c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743742109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743753808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408,"session_id":"4092f6f4-a6bb-4f94-950f-29c06dc41d9c"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743765537Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408} {"msg":"Activated sub_volume 77eb6b28-8fa1-46f0-96cb-4d513d302c56","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:36.743776345Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 205 len 20480 data: 4 5 5 6 6 02/30 Read block 140 len 36864 03/30 Write block 240 len 4096 data: 7 04/30 Write block 461 len 12288 data: 7 7 6 05/30 Read block 135 len 36864 06/30 Read block 177 len 36864 07/30 Write block 174 len 8192 data: 6 5 08/30 Write block 461 len 12288 data: 8 8 7 09/30 Read block 542 len 36864 10/30 Read block 113 len 36864 11/30 Write block 65 len 24576 data: 6 6 6 5 5 5 12/30 Write block 219 len 28672 data: 5 5 5 6 7 6 5 13/30 Read block 49 len 36864 14/30 Write block 462 len 24576 data: 9 8 7 5 6 5 15/30 Read block 438 len 24576 16/30 Write block 43 len 24576 data: 6 6 6 6 5 4 17/30 Write block 511 len 12288 data: 6 5 5 18/30 Read block 196 len 12288 19/30 Flush 20/30 Write block 201 len 4096 data: 4 21/30 Read block 380 len 8192 22/30 Flush 23/30 Write block 388 len 28672 data: 6 6 4 5 5 6 5 24/30 Write block 282 len 12288 data: 7 9 10 25/30 Read block 167 len 20480 26/30 Write block 556 len 16384 data: 4 4 5 3 27/30 Write block 245 len 16384 data: 6 4 5 4 28/30 Write block 248 len 12288 data: 5 4 4 29/30 Write block 408 len 28672 data: 8 7 9 8 6 5 5 30/30 Read block 432 len 16384 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 7 Done Sent Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 8 6 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:36.762968809Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 83 83 83 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 54 54 54 84 84 84 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 52 52 52 83 83 83 F F F 5 100-119 51 51 51 82 82 82 F F F 6 120-139 52 52 52 83 83 83 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 54 54 54 84 84 84 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 54 54 54 85 85 85 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 54 54 54 84 84 84 T T T 13 260-279 37 37 37 58 58 58 F F F 14 280-299 52 52 52 83 83 83 T T T 15 300-319 52 52 52 83 83 83 F F F 16 320-339 52 52 52 83 83 83 F F F 17 340-359 51 51 51 82 82 82 F F F 18 360-379 52 52 52 83 83 83 F F F 19 380-399 52 52 52 83 83 83 T T T 20 400-419 51 51 51 82 82 82 T T T 21 420-439 52 52 52 83 83 83 F F F 22 440-459 50 50 50 81 81 81 F F F 23 460-479 54 54 54 84 84 84 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 54 54 54 84 84 84 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 52 52 52 83 83 83 T T T 28 560-579 52 52 52 83 83 83 F F F 29 580-599 51 51 51 82 82 82 F F F Max gen: 54, Max flush: 85 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 83 83 83 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 54 54 54 84 84 84 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 52 52 52 83 83 83 F F F 5 100-119 51 51 51 82 82 82 F F F 6 120-139 52 52 52 83 83 83 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 54 54 54 84 84 84 F F F 9 180-199 46 46 46 73 73 73 F F F 10 200-219 54 54 54 85 85 85 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 37 37 37 58 58 58 F F F 14 280-299 55 55 55 86 86 86 F F F 15 300-319 52 52 52 83 83 83 F F F 16 320-339 52 52 52 83 83 83 F F F 17 340-359 51 51 51 82 82 82 F F F 18 360-379 52 52 52 83 83 83 F F F 19 380-399 55 55 55 86 86 86 F F F 20 400-419 55 55 55 86 86 86 F F F 21 420-439 52 52 52 83 83 83 F F F 22 440-459 50 50 50 81 81 81 F F F 23 460-479 54 54 54 84 84 84 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 54 54 54 84 84 84 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 55 55 55 86 86 86 F F F 28 560-579 52 52 52 83 83 83 F F F 29 580-599 51 51 51 82 82 82 F F F Max gen: 55, Max flush: 86 Begin loop 28 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.003316553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:51.044950899Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.045312051Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.076154441Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.076741089Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.076777525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.076790864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.076805092Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416} {"msg":"Crucible 57f00a5e-36c0-4c92-a918-a72cd48ea8de has session id: 4f6b1d5f-1106-4c10-ac45-7eb7641a71a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.077424057Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Upstairs opts: Upstairs UUID: 57f00a5e-36c0-4c92-a918-a72cd48ea8de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.077463093Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Crucible stats registered with UUID: 57f00a5e-36c0-4c92-a918-a72cd48ea8de","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.077481031Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:51.07749857Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.077939723Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.07797178Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416} {"msg":"57f00a5e-36c0-4c92-a918-a72cd48ea8de active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078029744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078056311Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"0","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078086478Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"0","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078101576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"1","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078124474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"1","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078149131Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"2","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078175818Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"2","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078190697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078206845Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078231202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51556)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.07834875Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42841)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078388836Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55462)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.078509973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.07920754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"1","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.079239557Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"2","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.079273583Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"0","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.079864391Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"1","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080015205Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"0","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080055421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"2","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.08044694Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080562078Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[0]R flush_numbers[0..12]: [83, 83, 84, 84, 83, 82, 83, 82, 84, 73, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.0806334Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[0]R generation[0..12]: [52, 52, 54, 54, 52, 51, 52, 51, 54, 46, 54, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080660577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080676956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[1]R flush_numbers[0..12]: [83, 83, 84, 84, 83, 82, 83, 82, 84, 73, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080691084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[1]R generation[0..12]: [52, 52, 54, 54, 52, 51, 52, 51, 54, 46, 54, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080705053Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080722521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[2]R flush_numbers[0..12]: [83, 83, 84, 84, 83, 82, 83, 82, 84, 73, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080737289Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[2]R generation[0..12]: [52, 52, 54, 54, 52, 51, 52, 51, 54, 46, 54, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080750498Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080764176Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080777525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080800233Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.0808235Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080848758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"":"downstairs","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080871335Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080884474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"0","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080897942Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"1","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080910701Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"client":"2","":"downstairs","session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.0809234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080942298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"57f00a5e-36c0-4c92-a918-a72cd48ea8de is now active with session: 4f6b1d5f-1106-4c10-ac45-7eb7641a71a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080963015Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080975874Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416,"session_id":"4f6b1d5f-1106-4c10-ac45-7eb7641a71a6"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.080990753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416} {"msg":"Activated sub_volume 57f00a5e-36c0-4c92-a918-a72cd48ea8de","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:51.081008861Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 542 len 20480 data: 3 4 5 4 5 02/30 Read block 293 len 12288 03/30 Read block 262 len 16384 04/30 Read block 567 len 28672 05/30 Read block 542 len 16384 06/30 Write block 368 len 24576 data: 6 6 6 6 6 6 07/30 Write block 323 len 16384 data: 5 7 4 4 08/30 Read block 1 len 36864 09/30 Write block 298 len 12288 data: 5 4 6 10/30 Write block 509 len 24576 data: 5 6 7 6 6 5 11/30 Read block 498 len 12288 12/30 Read block 217 len 20480 13/30 Write block 552 len 16384 data: 4 4 3 3 14/30 Write block 560 len 12288 data: 4 4 4 15/30 Read block 541 len 24576 16/30 Write block 88 len 24576 data: 4 3 3 3 3 4 17/30 Write block 402 len 20480 data: 7 6 6 6 7 18/30 Write block 481 len 36864 data: 6 6 5 6 6 6 6 6 5 19/30 Write block 45 len 24576 data: 7 7 6 5 5 5 20/30 Flush 21/30 Write block 582 len 32768 data: 5 6 9 8 8 8 7 6 22/30 Write block 179 len 8192 data: 4 5 23/30 Write block 170 len 32768 data: 4 3 5 5 7 6 4 4 24/30 Read block 275 len 32768 25/30 Write block 174 len 16384 data: 8 7 5 5 26/30 Write block 398 len 28672 data: 6 6 6 8 8 7 7 27/30 Read block 124 len 12288 28/30 Write block 261 len 36864 data: 5 5 5 5 6 6 5 8 7 29/30 Write block 17 len 12288 data: 4 3 4 30/30 Read block 26 len 16384 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 9 10 8 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:51.100691343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 83 83 83 T T T 1 020-039 52 52 52 83 83 83 F F F 2 040-059 56 56 56 87 87 87 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 51 51 51 82 82 82 F F F 6 120-139 52 52 52 83 83 83 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 54 54 54 84 84 84 T T T 9 180-199 46 46 46 73 73 73 T T T 10 200-219 54 54 54 85 85 85 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 37 37 37 58 58 58 T T T 14 280-299 56 56 56 87 87 87 F F F 15 300-319 56 56 56 87 87 87 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 51 51 51 82 82 82 F F F 18 360-379 56 56 56 87 87 87 F F F 19 380-399 55 55 55 86 86 86 T T T 20 400-419 56 56 56 87 87 87 T T T 21 420-439 52 52 52 83 83 83 F F F 22 440-459 50 50 50 81 81 81 F F F 23 460-479 54 54 54 84 84 84 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 56 56 56 87 87 87 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 56 56 56 87 87 87 F F F 28 560-579 56 56 56 87 87 87 F F F 29 580-599 51 51 51 82 82 82 T T T Max gen: 56, Max flush: 87 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 88 88 88 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 56 56 56 87 87 87 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 51 51 51 82 82 82 F F F 6 120-139 52 52 52 83 83 83 F F F 7 140-159 51 51 51 82 82 82 F F F 8 160-179 57 57 57 88 88 88 F F F 9 180-199 57 57 57 88 88 88 F F F 10 200-219 54 54 54 85 85 85 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 57 57 57 88 88 88 F F F 14 280-299 56 56 56 87 87 87 F F F 15 300-319 56 56 56 87 87 87 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 51 51 51 82 82 82 F F F 18 360-379 56 56 56 87 87 87 F F F 19 380-399 57 57 57 88 88 88 F F F 20 400-419 57 57 57 88 88 88 F F F 21 420-439 52 52 52 83 83 83 F F F 22 440-459 50 50 50 81 81 81 F F F 23 460-479 54 54 54 84 84 84 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 56 56 56 87 87 87 F F F 26 520-539 46 46 46 73 73 73 F F F 27 540-559 56 56 56 87 87 87 F F F 28 560-579 56 56 56 87 87 87 F F F 29 580-599 57 57 57 88 88 88 F F F Max gen: 57, Max flush: 88 Begin loop 29 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.376055276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.420633673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.421297103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.421331539Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.421351447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.421367166Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424} {"msg":"Crucible 755cfbbc-89ad-4262-a5be-47574313549d has session id: 840c2c51-41bf-4fb5-95e4-3c1b9074e4d1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.42198838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Upstairs opts: Upstairs UUID: 755cfbbc-89ad-4262-a5be-47574313549d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422034636Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Crucible stats registered with UUID: 755cfbbc-89ad-4262-a5be-47574313549d","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422056023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:55.422072261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422497317Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422528993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424} {"msg":"755cfbbc-89ad-4262-a5be-47574313549d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422619604Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422649291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"0","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422690137Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"0","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422717804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"1","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422789206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"1","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422826432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"2","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422863768Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"2","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422893245Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422920343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.42294353Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33442)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.422970017Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54059)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.423057248Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:41140)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.423099384Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.423734027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"0","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.423772443Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"1","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.423841946Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"2","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.424409266Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"0","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.424533653Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"1","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.424554241Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"2","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.424976906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425080845Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[0]R flush_numbers[0..12]: [88, 83, 87, 84, 87, 82, 83, 82, 88, 88, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425189894Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[0]R generation[0..12]: [57, 52, 56, 54, 56, 51, 52, 51, 57, 57, 54, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425208532Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.42522333Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[1]R flush_numbers[0..12]: [88, 83, 87, 84, 87, 82, 83, 82, 88, 88, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425236869Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[1]R generation[0..12]: [57, 52, 56, 54, 56, 51, 52, 51, 57, 57, 54, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425257027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425282354Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[2]R flush_numbers[0..12]: [88, 83, 87, 84, 87, 82, 83, 82, 88, 88, 85, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425308002Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[2]R generation[0..12]: [57, 52, 56, 54, 56, 51, 52, 51, 57, 57, 54, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425333649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425365076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425389603Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425404002Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Next flush: 89","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.42541901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425441188Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"":"downstairs","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425466095Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425487073Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"0","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425501021Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"1","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.42551368Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"client":"2","":"downstairs","session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425531658Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425553536Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"755cfbbc-89ad-4262-a5be-47574313549d is now active with session: 840c2c51-41bf-4fb5-95e4-3c1b9074e4d1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425577013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425598611Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424,"session_id":"840c2c51-41bf-4fb5-95e4-3c1b9074e4d1"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424} {"msg":"Activated sub_volume 755cfbbc-89ad-4262-a5be-47574313549d","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:55.425624948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 369 len 32768 data: 7 7 7 7 7 6 5 6 02/30 Read block 450 len 8192 03/30 Read block 49 len 8192 04/30 Write block 549 len 36864 data: 6 5 4 5 5 4 4 5 5 05/30 Write block 467 len 28672 data: 6 5 5 7 7 7 8 06/30 Read block 432 len 24576 07/30 Read block 254 len 12288 08/30 Write block 302 len 32768 data: 4 4 4 3 4 6 6 6 09/30 Write block 200 len 8192 data: 5 5 10/30 Flush 11/30 Flush 12/30 Write block 281 len 28672 data: 5 8 10 11 10 9 9 13/30 Read block 407 len 40960 14/30 Read block 372 len 36864 15/30 Write block 514 len 28672 data: 6 6 6 7 6 5 4 16/30 Write block 41 len 16384 data: 7 7 7 7 17/30 Write block 397 len 36864 data: 6 7 7 7 9 9 8 8 7 18/30 Flush 19/30 Write block 446 len 20480 data: 7 5 6 6 5 20/30 Read block 218 len 16384 21/30 Read block 519 len 16384 22/30 Write block 145 len 16384 data: 7 7 7 7 23/30 Read block 348 len 16384 24/30 Flush 25/30 Write block 427 len 16384 data: 4 4 7 9 26/30 Read block 351 len 40960 27/30 Read block 31 len 28672 28/30 Write block 385 len 4096 data: 6 29/30 Read block 203 len 28672 30/30 Write block 396 len 40960 data: 7 7 8 8 8 10 10 9 9 8 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:55.446155662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 88 88 88 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 58 58 58 91 91 91 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 51 51 51 82 82 82 F F F 6 120-139 52 52 52 83 83 83 F F F 7 140-159 58 58 58 92 92 92 F F F 8 160-179 57 57 57 88 88 88 F F F 9 180-199 57 57 57 88 88 88 F F F 10 200-219 58 58 58 89 89 89 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 57 57 57 88 88 88 F F F 14 280-299 58 58 58 91 91 91 F F F 15 300-319 58 58 58 89 89 89 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 51 51 51 82 82 82 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 58 58 58 91 91 91 T T T 20 400-419 58 58 58 91 91 91 T T T 21 420-439 52 52 52 83 83 83 T T T 22 440-459 58 58 58 92 92 92 F F F 23 460-479 58 58 58 89 89 89 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 58 58 58 91 91 91 F F F 26 520-539 58 58 58 91 91 91 F F F 27 540-559 58 58 58 89 89 89 F F F 28 560-579 56 56 56 87 87 87 F F F 29 580-599 57 57 57 88 88 88 F F F Max gen: 58, Max flush: 92 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 88 88 88 F F F 1 020-039 52 52 52 83 83 83 F F F 2 040-059 58 58 58 91 91 91 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 56 56 56 87 87 87 F F F 5 100-119 51 51 51 82 82 82 F F F 6 120-139 52 52 52 83 83 83 F F F 7 140-159 58 58 58 92 92 92 F F F 8 160-179 57 57 57 88 88 88 F F F 9 180-199 57 57 57 88 88 88 F F F 10 200-219 58 58 58 89 89 89 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 57 57 57 88 88 88 F F F 14 280-299 58 58 58 91 91 91 F F F 15 300-319 58 58 58 89 89 89 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 51 51 51 82 82 82 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 59 59 59 93 93 93 F F F 20 400-419 59 59 59 93 93 93 F F F 21 420-439 59 59 59 93 93 93 F F F 22 440-459 58 58 58 92 92 92 F F F 23 460-479 58 58 58 89 89 89 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 58 58 58 91 91 91 F F F 26 520-539 58 58 58 91 91 91 F F F 27 540-559 58 58 58 89 89 89 F F F 28 560-579 56 56 56 87 87 87 F F F 29 580-599 57 57 57 88 88 88 F F F Max gen: 59, Max flush: 93 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.676657359Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.721437374Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.722027292Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.722061469Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.722075887Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.722090466Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432} {"msg":"Crucible 736102b5-cd18-4f73-9a4f-05e735e41d77 has session id: 7c64208e-bbf4-4047-976d-4832c87ac8f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.722682674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Upstairs opts: Upstairs UUID: 736102b5-cd18-4f73-9a4f-05e735e41d77, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.722753436Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Crucible stats registered with UUID: 736102b5-cd18-4f73-9a4f-05e735e41d77","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.722781513Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:59.722804561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723231936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723257923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432} {"msg":"736102b5-cd18-4f73-9a4f-05e735e41d77 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723325966Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723344114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"0","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723377941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"0","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723392759Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"1","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723406878Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"1","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723419906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"2","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723433585Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"2","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723446323Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723467721Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723492518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56048)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723516066Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:47261)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723555642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46333)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.723584899Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.724299304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"1","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.724328831Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"0","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.724994761Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"1","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.725051195Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"0","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.725713275Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.725749381Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.725942521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"2","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.726454027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"2","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[0]R flush_numbers[0..12]: [88, 83, 91, 84, 87, 82, 83, 92, 88, 88, 89, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.726950375Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[0]R generation[0..12]: [57, 52, 58, 54, 56, 51, 52, 58, 57, 57, 58, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.726969743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.726983832Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[1]R flush_numbers[0..12]: [88, 83, 91, 84, 87, 82, 83, 92, 88, 88, 89, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.72699741Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[1]R generation[0..12]: [57, 52, 58, 54, 56, 51, 52, 58, 57, 57, 58, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727010739Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727024298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[2]R flush_numbers[0..12]: [88, 83, 91, 84, 87, 82, 83, 92, 88, 88, 89, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727037456Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[2]R generation[0..12]: [57, 52, 58, 54, 56, 51, 52, 58, 57, 57, 58, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727050795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727064223Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727080002Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.72709296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727114188Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727144005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"":"downstairs","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727168752Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727185281Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"0","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727206339Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"1","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727230526Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"client":"2","":"downstairs","session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727253994Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727267702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"736102b5-cd18-4f73-9a4f-05e735e41d77 is now active with session: 7c64208e-bbf4-4047-976d-4832c87ac8f0","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.72728953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:18:59.727311427Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432,"session_id":"7c64208e-bbf4-4047-976d-4832c87ac8f0"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727333325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432} {"msg":"Activated sub_volume 736102b5-cd18-4f73-9a4f-05e735e41d77","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:18:59.727352063Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 132 len 32768 data: 5 6 6 5 5 4 5 5 02/30 Write block 96 len 36864 data: 7 6 6 9 9 9 9 8 7 03/30 Write block 407 len 32768 data: 7 9 8 10 9 7 6 6 04/30 Write block 549 len 32768 data: 7 6 5 6 6 5 5 6 05/30 Write block 196 len 32768 data: 8 5 4 4 6 6 5 6 06/30 Write block 449 len 40960 data: 7 6 7 6 5 6 7 8 7 5 07/30 Read block 24 len 8192 08/30 Read block 86 len 32768 09/30 Read block 466 len 8192 10/30 Read block 521 len 24576 11/30 Write block 354 len 4096 data: 5 12/30 Read block 159 len 24576 13/30 Write block 558 len 40960 data: 6 4 5 5 5 3 3 3 4 2 14/30 Write block 560 len 40960 data: 6 6 6 4 4 4 5 3 3 4 15/30 Write block 108 len 20480 data: 5 5 5 5 4 16/30 Read block 235 len 24576 17/30 Read block 490 len 24576 18/30 Flush 19/30 Write block 141 len 16384 data: 5 7 8 8 20/30 Write block 134 len 4096 data: 7 21/30 Write block 548 len 40960 data: 6 8 7 6 7 7 6 6 7 6 22/30 Write block 595 len 20480 data: 4 4 3 3 2 23/30 Write block 39 len 36864 data: 4 5 8 8 8 8 8 8 7 24/30 Write block 449 len 40960 data: 8 7 8 7 6 7 8 9 8 6 25/30 Read block 164 len 32768 26/30 Write block 581 len 16384 data: 5 6 7 10 27/30 Read block 551 len 8192 28/30 Write block 341 len 4096 data: 6 29/30 Read block 389 len 20480 30/30 Write block 139 len 24576 data: 6 6 6 8 9 9 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 4 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:18:59.74689585Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 88 88 88 F F F 1 020-039 52 52 52 83 83 83 T T T 2 040-059 58 58 58 91 91 91 T T T 3 060-079 54 54 54 84 84 84 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 60 60 60 94 94 94 F F F 6 120-139 60 60 60 94 94 94 T T T 7 140-159 58 58 58 92 92 92 T T T 8 160-179 57 57 57 88 88 88 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 60 60 60 94 94 94 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 57 57 57 88 88 88 F F F 14 280-299 58 58 58 91 91 91 F F F 15 300-319 58 58 58 89 89 89 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 60 60 60 94 94 94 T T T 18 360-379 58 58 58 89 89 89 F F F 19 380-399 59 59 59 93 93 93 F F F 20 400-419 60 60 60 94 94 94 F F F 21 420-439 59 59 59 93 93 93 F F F 22 440-459 60 60 60 94 94 94 T T T 23 460-479 58 58 58 89 89 89 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 58 58 58 91 91 91 F F F 26 520-539 58 58 58 91 91 91 F F F 27 540-559 60 60 60 94 94 94 T T T 28 560-579 60 60 60 94 94 94 F F F 29 580-599 57 57 57 88 88 88 T T T Max gen: 60, Max flush: 94 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 88 88 88 F F F 1 020-039 61 61 61 95 95 95 F F F 2 040-059 61 61 61 95 95 95 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 60 60 60 94 94 94 F F F 6 120-139 61 61 61 95 95 95 F F F 7 140-159 61 61 61 95 95 95 F F F 8 160-179 57 57 57 88 88 88 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 60 60 60 94 94 94 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 57 57 57 88 88 88 F F F 14 280-299 58 58 58 91 91 91 F F F 15 300-319 58 58 58 89 89 89 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 59 59 59 93 93 93 F F F 20 400-419 60 60 60 94 94 94 F F F 21 420-439 59 59 59 93 93 93 F F F 22 440-459 61 61 61 95 95 95 F F F 23 460-479 58 58 58 89 89 89 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 58 58 58 91 91 91 F F F 26 520-539 58 58 58 91 91 91 F F F 27 540-559 61 61 61 95 95 95 F F F 28 560-579 60 60 60 94 94 94 F F F 29 580-599 61 61 61 95 95 95 F F F Max gen: 61, Max flush: 95 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:03.998845272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.042562659Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.043206471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.043241828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.043256706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.043271545Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440} {"msg":"Crucible b47275bf-4b4a-490b-92b8-353843d479df has session id: ca785fcc-d854-450f-a838-4c17c3c2e0b9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.043811338Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Upstairs opts: Upstairs UUID: b47275bf-4b4a-490b-92b8-353843d479df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.043843744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Crucible stats registered with UUID: b47275bf-4b4a-490b-92b8-353843d479df","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.043868732Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:04.043892319Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.04446005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044484667Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440} {"msg":"b47275bf-4b4a-490b-92b8-353843d479df active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.04455048Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044568888Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"0","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044597085Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"0","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044611784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"1","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044625802Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"1","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044638771Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044652459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044665218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044686406Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044716463Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50661)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044799214Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:32850)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.044833251Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:04.044865807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:04.044895624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:04.044979725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.045093273Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.045548685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"1","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.045584222Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"0","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.046219995Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"0","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.046248062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"1","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.046817632Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:04.046841249Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.046530413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56404)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.046671158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.047865683Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.04856061Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[0]R flush_numbers[0..12]: [88, 95, 95, 84, 94, 94, 95, 95, 88, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049125421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[0]R generation[0..12]: [57, 61, 61, 54, 60, 60, 61, 61, 57, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049147418Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049162617Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[1]R flush_numbers[0..12]: [88, 95, 95, 84, 94, 94, 95, 95, 88, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049176455Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[1]R generation[0..12]: [57, 61, 61, 54, 60, 60, 61, 61, 57, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049189904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049203692Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[2]R flush_numbers[0..12]: [88, 95, 95, 84, 94, 94, 95, 95, 88, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049216791Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[2]R generation[0..12]: [57, 61, 61, 54, 60, 60, 61, 61, 57, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.04923004Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049243718Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049256337Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049269496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049286134Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049299212Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"":"downstairs","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049312301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.04932406Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"0","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049337328Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"1","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049350097Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"client":"2","":"downstairs","session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049362856Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-18T15:19:14.049381814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"b47275bf-4b4a-490b-92b8-353843d479df is now active with session: ca785fcc-d854-450f-a838-4c17c3c2e0b9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049394672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049406311Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440,"session_id":"ca785fcc-d854-450f-a838-4c17c3c2e0b9"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.04941823Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440} {"msg":"Activated sub_volume b47275bf-4b4a-490b-92b8-353843d479df","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:14.049429249Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 591 len 24576 data: 3 2 3 3 5 5 02/30 Flush 03/30 Write block 124 len 4096 data: 3 04/30 Read block 143 len 20480 05/30 Read block 84 len 4096 06/30 Read block 333 len 12288 07/30 Write block 163 len 12288 data: 5 5 4 08/30 Read block 534 len 20480 09/30 Write block 573 len 40960 data: 6 4 4 3 3 3 2 2 6 7 10/30 Flush 11/30 Write block 542 len 4096 data: 4 12/30 Read block 351 len 16384 13/30 Write block 447 len 4096 data: 6 14/30 Read block 556 len 28672 15/30 Write block 510 len 24576 data: 7 8 7 7 7 7 16/30 Write block 465 len 36864 data: 6 7 7 6 6 8 8 8 9 17/30 Read block 327 len 24576 18/30 Write block 427 len 28672 data: 5 5 8 10 9 8 8 19/30 Write block 157 len 36864 data: 3 3 4 4 4 5 6 6 5 20/30 Read block 287 len 32768 21/30 Read block 554 len 36864 22/30 Write block 162 len 24576 data: 6 7 7 6 5 5 23/30 Write block 136 len 28672 data: 6 5 6 7 7 7 9 24/30 Write block 551 len 24576 data: 7 8 8 7 7 8 25/30 Write block 266 len 28672 data: 7 6 9 8 8 8 9 26/30 Read block 36 len 24576 27/30 Write block 167 len 36864 data: 6 5 4 5 4 6 6 9 8 28/30 Write block 578 len 16384 data: 4 3 3 7 29/30 Read block 552 len 8192 30/30 Write block 128 len 24576 data: 5 5 4 4 6 7 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 1 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Write 9 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:14.070003987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 88 88 88 F F F 1 020-039 61 61 61 95 95 95 F F F 2 040-059 61 61 61 95 95 95 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 60 60 60 94 94 94 F F F 6 120-139 62 62 62 97 97 97 T T T 7 140-159 61 61 61 95 95 95 T T T 8 160-179 62 62 62 97 97 97 T T T 9 180-199 60 60 60 94 94 94 F F F 10 200-219 60 60 60 94 94 94 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 57 57 57 88 88 88 T T T 14 280-299 58 58 58 91 91 91 F F F 15 300-319 58 58 58 89 89 89 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 59 59 59 93 93 93 F F F 20 400-419 60 60 60 94 94 94 F F F 21 420-439 59 59 59 93 93 93 T T T 22 440-459 61 61 61 95 95 95 T T T 23 460-479 58 58 58 89 89 89 T T T 24 480-499 56 56 56 87 87 87 F F F 25 500-519 58 58 58 91 91 91 T T T 26 520-539 58 58 58 91 91 91 F F F 27 540-559 61 61 61 95 95 95 T T T 28 560-579 62 62 62 97 97 97 T T T 29 580-599 62 62 62 97 97 97 T T T Max gen: 62, Max flush: 97 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 88 88 88 F F F 1 020-039 61 61 61 95 95 95 F F F 2 040-059 61 61 61 95 95 95 F F F 3 060-079 54 54 54 84 84 84 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 60 60 60 94 94 94 F F F 6 120-139 63 63 63 98 98 98 F F F 7 140-159 63 63 63 98 98 98 F F F 8 160-179 63 63 63 98 98 98 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 60 60 60 94 94 94 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 55 55 55 86 86 86 F F F 13 260-279 63 63 63 98 98 98 F F F 14 280-299 58 58 58 91 91 91 F F F 15 300-319 58 58 58 89 89 89 F F F 16 320-339 56 56 56 87 87 87 F F F 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 59 59 59 93 93 93 F F F 20 400-419 60 60 60 94 94 94 F F F 21 420-439 63 63 63 98 98 98 F F F 22 440-459 63 63 63 98 98 98 F F F 23 460-479 63 63 63 98 98 98 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 63 63 63 98 98 98 F F F 26 520-539 58 58 58 91 91 91 F F F 27 540-559 63 63 63 98 98 98 F F F 28 560-579 63 63 63 98 98 98 F F F 29 580-599 63 63 63 98 98 98 F F F Max gen: 63, Max flush: 98 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.342558948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.385719293Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.386339858Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.386380314Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.386397892Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.38641558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448} {"msg":"Crucible cdfe6c33-0a3f-4a07-89a0-7634be8abde8 has session id: e2762212-8fd8-46a9-ba98-796b219f8973","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.38699183Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Upstairs opts: Upstairs UUID: cdfe6c33-0a3f-4a07-89a0-7634be8abde8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387042845Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Crucible stats registered with UUID: cdfe6c33-0a3f-4a07-89a0-7634be8abde8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387071451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:28.387097369Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387681787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387710704Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448} {"msg":"cdfe6c33-0a3f-4a07-89a0-7634be8abde8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387786127Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387814013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"0","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387842321Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"0","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387856959Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"1","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387871298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"1","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387884186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387897905Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387910763Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387930101Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.387946499Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:28.388023242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:28.388057118Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"ds_connection connected from Ok(127.0.0.1:50319)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.388083065Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:28.388109702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"ds_connection connected from Ok(127.0.0.1:57520)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.38813211Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.388200593Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.388678023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"0","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.388717649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"1","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.389235584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"0","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.389339693Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"1","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.389708234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:28.389873297Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.389558364Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"ds_connection connected from Ok(127.0.0.1:59599)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.389729496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.390957067Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.391539455Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[0]R flush_numbers[0..12]: [88, 95, 95, 84, 94, 94, 98, 98, 98, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392118895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[0]R generation[0..12]: [57, 61, 61, 54, 60, 60, 63, 63, 63, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392148551Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.39216844Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[1]R flush_numbers[0..12]: [88, 95, 95, 84, 94, 94, 98, 98, 98, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392186227Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[1]R generation[0..12]: [57, 61, 61, 54, 60, 60, 63, 63, 63, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392203186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392220824Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[2]R flush_numbers[0..12]: [88, 95, 95, 84, 94, 94, 98, 98, 98, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392237942Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[2]R generation[0..12]: [57, 61, 61, 54, 60, 60, 63, 63, 63, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.39225454Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392271239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392287767Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392312724Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392328613Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.39235002Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"":"downstairs","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392366879Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392381657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"0","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392397925Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"1","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392414024Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"client":"2","":"downstairs","session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392430072Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392444851Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"cdfe6c33-0a3f-4a07-89a0-7634be8abde8 is now active with session: e2762212-8fd8-46a9-ba98-796b219f8973","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392460319Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392474777Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448,"session_id":"e2762212-8fd8-46a9-ba98-796b219f8973"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:19:38.392489566Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448} {"msg":"Activated sub_volume cdfe6c33-0a3f-4a07-89a0-7634be8abde8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:38.392503274Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 166 len 12288 data: 6 7 6 02/30 Write block 70 len 36864 data: 6 5 5 4 5 4 4 6 4 03/30 Read block 217 len 8192 04/30 Write block 69 len 36864 data: 6 7 6 6 5 6 5 5 7 05/30 Read block 158 len 32768 06/30 Read block 556 len 28672 07/30 Read block 179 len 32768 08/30 Read block 220 len 40960 09/30 Write block 462 len 40960 data: 10 9 8 7 8 8 7 7 9 9 10/30 Read block 284 len 20480 11/30 Write block 542 len 28672 data: 5 5 6 5 6 6 7 12/30 Read block 284 len 12288 13/30 Write block 548 len 24576 data: 8 9 8 8 9 9 14/30 Read block 316 len 8192 15/30 Read block 557 len 16384 16/30 Write block 399 len 8192 data: 9 9 17/30 Write block 251 len 20480 data: 4 4 4 4 4 18/30 Write block 278 len 32768 data: 4 6 5 6 9 11 12 11 19/30 Read block 371 len 20480 20/30 Flush 21/30 Flush 22/30 Write block 152 len 24576 data: 4 5 5 3 3 4 23/30 Write block 317 len 20480 data: 5 5 6 4 6 24/30 Read block 70 len 36864 25/30 Write block 47 len 28672 data: 8 6 6 6 4 5 4 26/30 Write block 260 len 4096 data: 3 27/30 Write block 298 len 28672 data: 6 5 7 6 5 5 5 28/30 Write block 526 len 32768 data: 4 4 4 2 2 2 3 4 29/30 Read block 512 len 16384 30/30 Write block 12 len 32768 data: 7 6 6 7 6 5 4 5 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:38.413498119Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 57 57 57 88 88 88 T T T 1 020-039 61 61 61 95 95 95 F F F 2 040-059 61 61 61 95 95 95 T T T 3 060-079 64 64 64 99 99 99 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 60 60 60 94 94 94 F F F 6 120-139 63 63 63 98 98 98 F F F 7 140-159 63 63 63 98 98 98 T T T 8 160-179 64 64 64 99 99 99 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 60 60 60 94 94 94 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 64 64 64 99 99 99 F F F 13 260-279 64 64 64 99 99 99 T T T 14 280-299 64 64 64 99 99 99 T T T 15 300-319 58 58 58 89 89 89 T T T 16 320-339 56 56 56 87 87 87 T T T 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 64 64 64 99 99 99 F F F 20 400-419 64 64 64 99 99 99 F F F 21 420-439 63 63 63 98 98 98 F F F 22 440-459 63 63 63 98 98 98 F F F 23 460-479 64 64 64 99 99 99 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 63 63 63 98 98 98 F F F 26 520-539 58 58 58 91 91 91 T T T 27 540-559 64 64 64 99 99 99 F F F 28 560-579 63 63 63 98 98 98 F F F 29 580-599 63 63 63 98 98 98 F F F Max gen: 64, Max flush: 99 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 100 100 100 F F F 1 020-039 61 61 61 95 95 95 F F F 2 040-059 65 65 65 100 100 100 F F F 3 060-079 64 64 64 99 99 99 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 60 60 60 94 94 94 F F F 6 120-139 63 63 63 98 98 98 F F F 7 140-159 65 65 65 100 100 100 F F F 8 160-179 64 64 64 99 99 99 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 60 60 60 94 94 94 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 64 64 64 99 99 99 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 65 65 65 100 100 100 F F F 15 300-319 65 65 65 100 100 100 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 64 64 64 99 99 99 F F F 20 400-419 64 64 64 99 99 99 F F F 21 420-439 63 63 63 98 98 98 F F F 22 440-459 63 63 63 98 98 98 F F F 23 460-479 64 64 64 99 99 99 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 63 63 63 98 98 98 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 64 64 64 99 99 99 F F F 28 560-579 63 63 63 98 98 98 F F F 29 580-599 63 63 63 98 98 98 F F F Max gen: 65, Max flush: 100 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.669450084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.713091509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.71375383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.713789416Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.713804754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.713819813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456} {"msg":"Crucible f10a1ca9-62e9-476d-9265-b1bc6c8a5c42 has session id: 90fe0621-cab9-41f7-ab07-cab3c72e51d4","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.714525158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Upstairs opts: Upstairs UUID: f10a1ca9-62e9-476d-9265-b1bc6c8a5c42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.714559535Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Crucible stats registered with UUID: f10a1ca9-62e9-476d-9265-b1bc6c8a5c42","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.714577783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:52.714593271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715150843Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715184039Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456} {"msg":"f10a1ca9-62e9-476d-9265-b1bc6c8a5c42 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715237874Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715264601Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"0","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715296957Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"0","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715313076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"1","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715327334Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"1","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715340473Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"2","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715398787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"2","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715494297Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715528323Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.71565375Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:50077)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715776477Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65485)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715801635Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62362)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.715824622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.716168556Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"2","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.716196423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"0","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.716816508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"2","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.716950624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"0","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.717480868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.717545181Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.717996194Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"1","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.71850959Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"1","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[0]R flush_numbers[0..12]: [100, 95, 100, 99, 94, 94, 98, 100, 99, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719027126Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[0]R generation[0..12]: [65, 61, 65, 64, 60, 60, 63, 65, 64, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719057102Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719075811Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[1]R flush_numbers[0..12]: [100, 95, 100, 99, 94, 94, 98, 100, 99, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719092769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[1]R generation[0..12]: [65, 61, 65, 64, 60, 60, 63, 65, 64, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719109127Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719126255Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[2]R flush_numbers[0..12]: [100, 95, 100, 99, 94, 94, 98, 100, 99, 94, 94, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719146803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[2]R generation[0..12]: [65, 61, 65, 64, 60, 60, 63, 65, 64, 60, 60, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719163761Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719181209Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719197038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719213076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:19:52.719228815Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719248022Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"":"downstairs","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719263891Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719278399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"0","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719295568Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"1","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719311586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"client":"2","":"downstairs","session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719327404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719342232Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"f10a1ca9-62e9-476d-9265-b1bc6c8a5c42 is now active with session: 90fe0621-cab9-41f7-ab07-cab3c72e51d4","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719357911Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719372369Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456,"session_id":"90fe0621-cab9-41f7-ab07-cab3c72e51d4"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719386978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456} {"msg":"Activated sub_volume f10a1ca9-62e9-476d-9265-b1bc6c8a5c42","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:52.719403306Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 402 len 20480 data: 11 10 10 9 8 02/30 Write block 130 len 40960 data: 5 5 7 8 8 6 7 6 7 8 03/30 Flush 04/30 Write block 543 len 16384 data: 6 7 6 7 05/30 Write block 543 len 20480 data: 7 8 7 8 7 06/30 Write block 163 len 8192 data: 8 8 07/30 Flush 08/30 Write block 459 len 12288 data: 7 7 9 09/30 Flush 10/30 Write block 280 len 8192 data: 6 7 11/30 Write block 200 len 12288 data: 7 7 6 12/30 Write block 447 len 32768 data: 7 7 9 8 9 8 7 8 13/30 Write block 507 len 36864 data: 4 4 6 8 9 8 8 8 8 14/30 Read block 423 len 4096 15/30 Read block 577 len 36864 16/30 Write block 251 len 20480 data: 5 5 5 5 5 17/30 Read block 569 len 40960 18/30 Write block 116 len 20480 data: 3 2 2 3 4 19/30 Write block 152 len 28672 data: 5 6 6 4 4 5 4 20/30 Write block 63 len 28672 data: 8 7 7 7 7 6 7 21/30 Flush 22/30 Write block 456 len 24576 data: 10 9 7 8 8 10 23/30 Read block 60 len 32768 24/30 Write block 137 len 20480 data: 7 8 9 8 8 25/30 Read block 276 len 16384 26/30 Write block 62 len 20480 data: 7 9 8 8 8 27/30 Write block 560 len 8192 data: 7 7 28/30 Read block 18 len 16384 29/30 Write block 31 len 28672 data: 6 6 6 5 6 5 6 30/30 Read block 307 len 16384 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 8 9 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:52.738653474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 100 100 100 F F F 1 020-039 61 61 61 95 95 95 T T T 2 040-059 65 65 65 100 100 100 F F F 3 060-079 66 66 66 104 104 104 T T T 4 080-099 60 60 60 94 94 94 F F F 5 100-119 66 66 66 104 104 104 F F F 6 120-139 66 66 66 104 104 104 T T T 7 140-159 66 66 66 104 104 104 T T T 8 160-179 66 66 66 102 102 102 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 66 66 66 104 104 104 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 66 66 66 104 104 104 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 65 65 65 100 100 100 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 64 64 64 99 99 99 F F F 20 400-419 66 66 66 101 101 101 F F F 21 420-439 63 63 63 98 98 98 F F F 22 440-459 66 66 66 104 104 104 T T T 23 460-479 66 66 66 103 103 103 T T T 24 480-499 56 56 56 87 87 87 F F F 25 500-519 66 66 66 104 104 104 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 66 66 66 102 102 102 F F F 28 560-579 63 63 63 98 98 98 T T T 29 580-599 63 63 63 98 98 98 F F F Max gen: 66, Max flush: 104 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 100 100 100 F F F 1 020-039 67 67 67 105 105 105 F F F 2 040-059 65 65 65 100 100 100 F F F 3 060-079 67 67 67 105 105 105 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 66 66 66 104 104 104 F F F 6 120-139 67 67 67 105 105 105 F F F 7 140-159 67 67 67 105 105 105 F F F 8 160-179 66 66 66 102 102 102 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 66 66 66 104 104 104 F F F 11 220-239 54 54 54 84 84 84 F F F 12 240-259 66 66 66 104 104 104 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 65 65 65 100 100 100 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 64 64 64 99 99 99 F F F 20 400-419 66 66 66 101 101 101 F F F 21 420-439 63 63 63 98 98 98 F F F 22 440-459 67 67 67 105 105 105 F F F 23 460-479 67 67 67 105 105 105 F F F 24 480-499 56 56 56 87 87 87 F F F 25 500-519 66 66 66 104 104 104 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 66 66 66 102 102 102 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 63 63 63 98 98 98 F F F Max gen: 67, Max flush: 105 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:56.989537399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.033832685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.034425613Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.034466849Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.034489576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.034508684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464} {"msg":"Crucible 6bf902c2-dd62-451b-830d-133e95675b9d has session id: 7f706227-468a-436b-a340-0ff5d06bdfcc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035041328Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Upstairs opts: Upstairs UUID: 6bf902c2-dd62-451b-830d-133e95675b9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035083194Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Crucible stats registered with UUID: 6bf902c2-dd62-451b-830d-133e95675b9d","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035108661Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:57.035129089Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035569003Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035603069Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464} {"msg":"6bf902c2-dd62-451b-830d-133e95675b9d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035656364Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035680591Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"0","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035723227Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"0","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035752913Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"1","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.03578457Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"1","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035810918Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"2","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035840184Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"2","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035872231Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035897238Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035918106Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:60659)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035943513Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34505)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035967801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:38984)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.035992388Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.036560999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"0","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.036681656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"1","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.03711797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"0","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.03721923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"1","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.03759269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.037801258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.037828586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"2","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038376758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"2","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[0]R flush_numbers[0..12]: [100, 105, 100, 105, 94, 104, 105, 105, 102, 94, 104, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038872776Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[0]R generation[0..12]: [65, 67, 65, 67, 60, 66, 67, 67, 66, 60, 66, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038896033Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038910502Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[1]R flush_numbers[0..12]: [100, 105, 100, 105, 94, 104, 105, 105, 102, 94, 104, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.03892424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[1]R generation[0..12]: [65, 67, 65, 67, 60, 66, 67, 67, 66, 60, 66, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038937829Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038951477Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[2]R flush_numbers[0..12]: [100, 105, 100, 105, 94, 104, 105, 105, 102, 94, 104, 84]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038964946Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[2]R generation[0..12]: [65, 67, 65, 67, 60, 66, 67, 67, 66, 60, 66, 54]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038978435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.038992163Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039013611Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:19:57.039037488Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039065176Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039091383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"":"downstairs","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039107521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.03911955Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"0","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039143987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"1","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039169605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"client":"2","":"downstairs","session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039189623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.03921185Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"6bf902c2-dd62-451b-830d-133e95675b9d is now active with session: 7f706227-468a-436b-a340-0ff5d06bdfcc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039235698Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039257745Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464,"session_id":"7f706227-468a-436b-a340-0ff5d06bdfcc"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039270974Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464} {"msg":"Activated sub_volume 6bf902c2-dd62-451b-830d-133e95675b9d","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:19:57.039283273Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 590 len 12288 data: 4 4 3 02/30 Write block 413 len 20480 data: 7 7 6 8 9 03/30 Read block 519 len 32768 04/30 Write block 33 len 16384 data: 7 6 7 6 05/30 Write block 218 len 16384 data: 6 6 6 6 06/30 Write block 208 len 4096 data: 7 07/30 Flush 08/30 Flush 09/30 Write block 414 len 4096 data: 8 10/30 Read block 60 len 16384 11/30 Write block 246 len 28672 data: 5 6 6 5 5 6 6 12/30 Write block 385 len 8192 data: 7 5 13/30 Read block 552 len 40960 14/30 Read block 421 len 28672 15/30 Read block 23 len 4096 16/30 Read block 316 len 28672 17/30 Read block 481 len 16384 18/30 Read block 227 len 16384 19/30 Write block 306 len 40960 data: 5 7 7 7 4 6 6 6 8 6 20/30 Write block 17 len 28672 data: 6 5 6 5 5 4 4 21/30 Write block 462 len 36864 data: 11 10 9 8 9 9 8 8 10 22/30 Read block 359 len 8192 23/30 Write block 148 len 12288 data: 8 5 5 24/30 Read block 451 len 36864 25/30 Write block 495 len 16384 data: 4 4 4 3 26/30 Read block 192 len 16384 27/30 Read block 323 len 40960 28/30 Read block 68 len 20480 29/30 Read block 307 len 8192 30/30 Read block 130 len 8192 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 1 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 2 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:19:57.058582265Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 65 65 65 100 100 100 T T T 1 020-039 68 68 68 106 106 106 T T T 2 040-059 65 65 65 100 100 100 F F F 3 060-079 67 67 67 105 105 105 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 66 66 66 104 104 104 F F F 6 120-139 67 67 67 105 105 105 F F F 7 140-159 67 67 67 105 105 105 T T T 8 160-179 66 66 66 102 102 102 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 68 68 68 106 106 106 F F F 11 220-239 68 68 68 106 106 106 F F F 12 240-259 66 66 66 104 104 104 T T T 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 65 65 65 100 100 100 T T T 16 320-339 65 65 65 100 100 100 F F F 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 64 64 64 99 99 99 T T T 20 400-419 68 68 68 106 106 106 T T T 21 420-439 63 63 63 98 98 98 F F F 22 440-459 67 67 67 105 105 105 F F F 23 460-479 67 67 67 105 105 105 T T T 24 480-499 56 56 56 87 87 87 T T T 25 500-519 66 66 66 104 104 104 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 66 66 66 102 102 102 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 68 68 68 106 106 106 F F F Max gen: 68, Max flush: 106 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 107 107 107 F F F 1 020-039 69 69 69 107 107 107 F F F 2 040-059 65 65 65 100 100 100 F F F 3 060-079 67 67 67 105 105 105 F F F 4 080-099 60 60 60 94 94 94 F F F 5 100-119 66 66 66 104 104 104 F F F 6 120-139 67 67 67 105 105 105 F F F 7 140-159 69 69 69 107 107 107 F F F 8 160-179 66 66 66 102 102 102 F F F 9 180-199 60 60 60 94 94 94 F F F 10 200-219 68 68 68 106 106 106 F F F 11 220-239 68 68 68 106 106 106 F F F 12 240-259 69 69 69 107 107 107 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 69 69 69 107 107 107 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 61 61 61 95 95 95 F F F 18 360-379 58 58 58 89 89 89 F F F 19 380-399 69 69 69 107 107 107 F F F 20 400-419 69 69 69 107 107 107 F F F 21 420-439 63 63 63 98 98 98 F F F 22 440-459 67 67 67 105 105 105 F F F 23 460-479 69 69 69 107 107 107 F F F 24 480-499 69 69 69 107 107 107 F F F 25 500-519 66 66 66 104 104 104 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 66 66 66 102 102 102 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 68 68 68 106 106 106 F F F Max gen: 69, Max flush: 107 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.318164459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:11.360877791Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.361292218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.392384122Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.392948462Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.392989368Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.393024924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.393043662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472} {"msg":"Crucible cbe2e391-4866-4701-be59-946e035ba35f has session id: 9f177e9b-f616-486b-9944-6b2c37768435","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.393596194Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Upstairs opts: Upstairs UUID: cbe2e391-4866-4701-be59-946e035ba35f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.393626901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Crucible stats registered with UUID: cbe2e391-4866-4701-be59-946e035ba35f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.393647509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:11.393662897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394049327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394073164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472} {"msg":"cbe2e391-4866-4701-be59-946e035ba35f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394156086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394173174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"0","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.39420321Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"0","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394228468Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"1","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394255195Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"1","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394280822Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"2","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394309029Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"2","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394333727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394358774Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394386791Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"ds_connection connected from Ok(127.0.0.1:48396)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394414318Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"ds_connection connected from Ok(127.0.0.1:61951)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394445985Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"ds_connection connected from Ok(127.0.0.1:65254)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.394474902Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.395053471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"1","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.395093387Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"2","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.395267799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"0","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.395724511Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"2","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.395753808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"1","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.39601432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"0","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396335586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396361224Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[0]R flush_numbers[0..12]: [107, 107, 100, 105, 94, 104, 105, 107, 102, 94, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396572912Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[0]R generation[0..12]: [69, 69, 65, 67, 60, 66, 67, 69, 66, 60, 68, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396593739Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396608598Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[1]R flush_numbers[0..12]: [107, 107, 100, 105, 94, 104, 105, 107, 102, 94, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396622326Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[1]R generation[0..12]: [69, 69, 65, 67, 60, 66, 67, 69, 66, 60, 68, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396640205Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396654283Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[2]R flush_numbers[0..12]: [107, 107, 100, 105, 94, 104, 105, 107, 102, 94, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396667742Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[2]R generation[0..12]: [69, 69, 65, 67, 60, 66, 67, 69, 66, 60, 68, 68]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.39668098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396694539Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396707287Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396720016Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396732465Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396751883Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"":"downstairs","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396765201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.39677697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"0","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396789889Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"1","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396802577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"client":"2","":"downstairs","session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396815406Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396828405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"cbe2e391-4866-4701-be59-946e035ba35f is now active with session: 9f177e9b-f616-486b-9944-6b2c37768435","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396843483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.396855412Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472,"session_id":"9f177e9b-f616-486b-9944-6b2c37768435"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:11.39686821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472} {"msg":"Activated sub_volume cbe2e391-4866-4701-be59-946e035ba35f","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:20:11.396879189Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 428 len 32768 data: 6 9 11 10 9 9 8 8 02/30 Write block 243 len 12288 data: 6 6 7 03/30 Read block 383 len 20480 04/30 Write block 359 len 16384 data: 3 3 4 6 05/30 Read block 403 len 28672 06/30 Read block 348 len 24576 07/30 Write block 227 len 32768 data: 4 3 3 3 6 6 4 4 08/30 Flush 09/30 Write block 87 len 28672 data: 4 5 4 4 4 4 5 10/30 Write block 485 len 16384 data: 7 7 7 7 11/30 Read block 483 len 4096 12/30 Write block 436 len 40960 data: 5 5 6 4 4 3 3 5 5 5 13/30 Write block 468 len 8192 data: 9 9 14/30 Write block 229 len 36864 data: 4 4 7 7 5 5 3 3 3 15/30 Write block 49 len 36864 data: 7 7 5 6 5 6 6 7 8 16/30 Read block 204 len 40960 17/30 Read block 329 len 28672 18/30 Write block 6 len 12288 data: 4 5 4 19/30 Write block 254 len 8192 data: 6 6 20/30 Flush 21/30 Write block 163 len 36864 data: 9 9 7 7 8 7 5 6 5 22/30 Write block 490 len 20480 data: 5 3 5 5 5 23/30 Read block 558 len 16384 24/30 Read block 543 len 16384 25/30 Read block 139 len 24576 26/30 Write block 248 len 32768 data: 7 6 6 7 7 6 7 7 27/30 Read block 544 len 24576 28/30 Write block 183 len 36864 data: 7 8 7 8 6 4 4 5 8 29/30 Read block 123 len 36864 30/30 Read block 379 len 36864 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 9 10 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:11.418081752Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 109 109 109 F F F 1 020-039 69 69 69 107 107 107 F F F 2 040-059 70 70 70 109 109 109 F F F 3 060-079 67 67 67 105 105 105 F F F 4 080-099 70 70 70 109 109 109 F F F 5 100-119 66 66 66 104 104 104 F F F 6 120-139 67 67 67 105 105 105 F F F 7 140-159 69 69 69 107 107 107 F F F 8 160-179 66 66 66 102 102 102 T T T 9 180-199 60 60 60 94 94 94 T T T 10 200-219 68 68 68 106 106 106 F F F 11 220-239 70 70 70 109 109 109 F F F 12 240-259 70 70 70 109 109 109 T T T 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 69 69 69 107 107 107 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 F F F 18 360-379 70 70 70 108 108 108 F F F 19 380-399 69 69 69 107 107 107 F F F 20 400-419 69 69 69 107 107 107 F F F 21 420-439 70 70 70 109 109 109 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 70 70 70 109 109 109 F F F 24 480-499 70 70 70 109 109 109 T T T 25 500-519 66 66 66 104 104 104 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 66 66 66 102 102 102 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 68 68 68 106 106 106 F F F Max gen: 70, Max flush: 109 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 109 109 109 F F F 1 020-039 69 69 69 107 107 107 F F F 2 040-059 70 70 70 109 109 109 F F F 3 060-079 67 67 67 105 105 105 F F F 4 080-099 70 70 70 109 109 109 F F F 5 100-119 66 66 66 104 104 104 F F F 6 120-139 67 67 67 105 105 105 F F F 7 140-159 69 69 69 107 107 107 F F F 8 160-179 71 71 71 110 110 110 F F F 9 180-199 71 71 71 110 110 110 F F F 10 200-219 68 68 68 106 106 106 F F F 11 220-239 70 70 70 109 109 109 F F F 12 240-259 71 71 71 110 110 110 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 69 69 69 107 107 107 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 F F F 18 360-379 70 70 70 108 108 108 F F F 19 380-399 69 69 69 107 107 107 F F F 20 400-419 69 69 69 107 107 107 F F F 21 420-439 70 70 70 109 109 109 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 70 70 70 109 109 109 F F F 24 480-499 71 71 71 110 110 110 F F F 25 500-519 66 66 66 104 104 104 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 66 66 66 102 102 102 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 68 68 68 106 106 106 F F F Max gen: 71, Max flush: 110 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.679738005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.724464606Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.72508691Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.725120417Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.725134405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.725148444Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480} {"msg":"Crucible 658902a7-946a-44c4-8cc3-394b10717eaf has session id: 773fb38a-2269-4311-bce9-4a2400b87ab1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.725686618Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Upstairs opts: Upstairs UUID: 658902a7-946a-44c4-8cc3-394b10717eaf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.725803775Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Crucible stats registered with UUID: 658902a7-946a-44c4-8cc3-394b10717eaf","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.725832582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:15.725846901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726471465Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726505801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480} {"msg":"658902a7-946a-44c4-8cc3-394b10717eaf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726591923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.72661329Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"0","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726641287Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"0","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726655186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"1","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726669044Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"1","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726686062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"2","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726700021Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"2","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.72671309Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726734008Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726759215Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47345)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726784502Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:41628)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.726817269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"ds_connection connected from Ok(127.0.0.1:63710)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.727020657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.727532304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"0","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.727571589Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"2","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.727710875Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"1","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.728240569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"2","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.728270706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"0","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.72842348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"1","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.72880149Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.72889525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[0]R flush_numbers[0..12]: [109, 107, 109, 105, 109, 104, 105, 107, 110, 110, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729004999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[0]R generation[0..12]: [70, 69, 70, 67, 70, 66, 67, 69, 71, 71, 68, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729025437Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729040325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[1]R flush_numbers[0..12]: [109, 107, 109, 105, 109, 104, 105, 107, 110, 110, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729054334Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[1]R generation[0..12]: [70, 69, 70, 67, 70, 66, 67, 69, 71, 71, 68, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729067482Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729081091Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[2]R flush_numbers[0..12]: [109, 107, 109, 105, 109, 104, 105, 107, 110, 110, 106, 109]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729094409Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[2]R generation[0..12]: [70, 69, 70, 67, 70, 66, 67, 69, 71, 71, 68, 70]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729107468Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729120917Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729133565Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729146364Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729158783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729175511Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"":"downstairs","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729196259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729208337Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"0","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729221296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"1","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729233865Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"client":"2","":"downstairs","session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729246544Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729258462Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"658902a7-946a-44c4-8cc3-394b10717eaf is now active with session: 773fb38a-2269-4311-bce9-4a2400b87ab1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729271221Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.72928322Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480,"session_id":"773fb38a-2269-4311-bce9-4a2400b87ab1"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729295208Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480} {"msg":"Activated sub_volume 658902a7-946a-44c4-8cc3-394b10717eaf","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:15.729306177Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 59 len 16384 data: 7 7 6 8 03/30 Read block 338 len 8192 04/30 Write block 242 len 24576 data: 6 7 7 8 6 7 05/30 Write block 461 len 16384 data: 11 12 11 10 06/30 Read block 490 len 28672 07/30 Flush 08/30 Write block 166 len 28672 data: 8 9 8 6 7 6 7 09/30 Read block 139 len 4096 10/30 Read block 291 len 28672 11/30 Write block 376 len 36864 data: 7 7 7 7 6 6 5 6 7 12/30 Read block 42 len 36864 13/30 Read block 295 len 12288 14/30 Read block 68 len 8192 15/30 Flush 16/30 Write block 511 len 32768 data: 10 9 9 9 9 7 8 7 17/30 Read block 345 len 28672 18/30 Read block 290 len 4096 19/30 Flush 20/30 Write block 216 len 40960 data: 8 8 7 7 7 7 7 8 7 6 21/30 Read block 383 len 12288 22/30 Read block 457 len 40960 23/30 Read block 166 len 40960 24/30 Write block 425 len 32768 data: 4 5 6 7 10 12 11 10 25/30 Write block 219 len 20480 data: 8 8 8 8 9 26/30 Write block 592 len 4096 data: 4 27/30 Read block 99 len 8192 28/30 Read block 106 len 16384 29/30 Read block 433 len 40960 30/30 Read block 491 len 40960 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 10 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:15.750756434Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 109 109 109 F F F 1 020-039 69 69 69 107 107 107 F F F 2 040-059 72 72 72 112 112 112 F F F 3 060-079 72 72 72 112 112 112 F F F 4 080-099 70 70 70 109 109 109 F F F 5 100-119 66 66 66 104 104 104 F F F 6 120-139 67 67 67 105 105 105 F F F 7 140-159 69 69 69 107 107 107 F F F 8 160-179 72 72 72 113 113 113 F F F 9 180-199 71 71 71 110 110 110 F F F 10 200-219 68 68 68 106 106 106 T T T 11 220-239 70 70 70 109 109 109 T T T 12 240-259 72 72 72 112 112 112 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 69 69 69 107 107 107 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 69 69 69 107 107 107 F F F 21 420-439 70 70 70 109 109 109 T T T 22 440-459 70 70 70 109 109 109 F F F 23 460-479 72 72 72 112 112 112 F F F 24 480-499 71 71 71 110 110 110 F F F 25 500-519 72 72 72 114 114 114 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 66 66 66 102 102 102 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 68 68 68 106 106 106 T T T Max gen: 72, Max flush: 114 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 70 70 70 109 109 109 F F F 1 020-039 69 69 69 107 107 107 F F F 2 040-059 72 72 72 112 112 112 F F F 3 060-079 72 72 72 112 112 112 F F F 4 080-099 70 70 70 109 109 109 F F F 5 100-119 66 66 66 104 104 104 F F F 6 120-139 67 67 67 105 105 105 F F F 7 140-159 69 69 69 107 107 107 F F F 8 160-179 72 72 72 113 113 113 F F F 9 180-199 71 71 71 110 110 110 F F F 10 200-219 73 73 73 115 115 115 F F F 11 220-239 73 73 73 115 115 115 F F F 12 240-259 72 72 72 112 112 112 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 69 69 69 107 107 107 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 69 69 69 107 107 107 F F F 21 420-439 73 73 73 115 115 115 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 72 72 72 112 112 112 F F F 24 480-499 71 71 71 110 110 110 F F F 25 500-519 72 72 72 114 114 114 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 66 66 66 102 102 102 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 73 73 73 115 115 115 F F F Max gen: 73, Max flush: 115 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:19.985316413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.029373135Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.030018977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.030054603Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.030069282Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.03008383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488} {"msg":"Crucible f4ce153f-2be2-4138-a378-9b1e03492f65 has session id: b29b6150-7b8b-46a8-93d1-80149e8d1d11","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.030589197Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Upstairs opts: Upstairs UUID: f4ce153f-2be2-4138-a378-9b1e03492f65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.030631043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Crucible stats registered with UUID: f4ce153f-2be2-4138-a378-9b1e03492f65","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.03065678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:20.030677428Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031172806Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031208242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488} {"msg":"f4ce153f-2be2-4138-a378-9b1e03492f65 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031283914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031302612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"0","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031332669Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"0","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031346767Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031360676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031373414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"2","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031387053Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"2","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031399832Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.03141987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031444697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"ds_connection connected from Ok(127.0.0.1:62027)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031614799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"ds_connection connected from Ok(127.0.0.1:53182)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031746945Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:20.031789211Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:20.031823697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:20.031855524Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.031937135Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.032388248Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"2","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.032415925Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"0","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.03303272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"0","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.033061147Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"2","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.033597981Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:20.033694991Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.033515611Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"ds_connection connected from Ok(127.0.0.1:47325)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.033804371Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.034856111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.035409462Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[0]R flush_numbers[0..12]: [109, 107, 112, 112, 109, 104, 105, 107, 113, 110, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.035972033Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[0]R generation[0..12]: [70, 69, 72, 72, 70, 66, 67, 69, 72, 71, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.03600396Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036023638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[1]R flush_numbers[0..12]: [109, 107, 112, 112, 109, 104, 105, 107, 113, 110, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036041306Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[1]R generation[0..12]: [70, 69, 72, 72, 70, 66, 67, 69, 72, 71, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036057954Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036074992Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[2]R flush_numbers[0..12]: [109, 107, 112, 112, 109, 104, 105, 107, 113, 110, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036091831Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[2]R generation[0..12]: [70, 69, 72, 72, 70, 66, 67, 69, 72, 71, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036108259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036125007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036142045Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036167413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Next flush: 116","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036183261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.03619911Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"":"downstairs","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036215248Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036229806Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"0","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036248494Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"1","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036264783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"client":"2","":"downstairs","session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036281011Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036296239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"f4ce153f-2be2-4138-a378-9b1e03492f65 is now active with session: b29b6150-7b8b-46a8-93d1-80149e8d1d11","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036315767Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036330725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488,"session_id":"b29b6150-7b8b-46a8-93d1-80149e8d1d11"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036345764Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488} {"msg":"Activated sub_volume f4ce153f-2be2-4138-a378-9b1e03492f65","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:30.036359393Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 8192 data: 7 7 02/30 Write block 139 len 32768 data: 10 9 9 10 10 10 8 8 03/30 Write block 100 len 20480 data: 10 10 10 9 8 04/30 Flush 05/30 Write block 12 len 28672 data: 8 7 7 8 8 8 6 06/30 Read block 446 len 12288 07/30 Read block 543 len 40960 08/30 Read block 278 len 20480 09/30 Write block 6 len 24576 data: 5 6 5 5 5 6 10/30 Read block 370 len 12288 11/30 Read block 541 len 16384 12/30 Write block 551 len 28672 data: 9 10 10 8 8 9 7 13/30 Read block 151 len 24576 14/30 Write block 126 len 12288 data: 3 3 6 15/30 Read block 378 len 4096 16/30 Read block 29 len 8192 17/30 Write block 157 len 28672 data: 6 5 5 5 5 7 10 18/30 Read block 481 len 36864 19/30 Write block 404 len 24576 data: 11 10 9 8 10 9 20/30 Read block 222 len 40960 21/30 Read block 179 len 28672 22/30 Write block 54 len 24576 data: 7 7 8 9 8 8 23/30 Write block 186 len 16384 data: 9 7 5 5 24/30 Write block 315 len 4096 data: 7 25/30 Read block 185 len 28672 26/30 Flush 27/30 Write block 34 len 16384 data: 7 8 7 7 28/30 Write block 174 len 40960 data: 10 9 6 6 4 5 6 6 6 8 29/30 Read block 142 len 16384 30/30 Read block 271 len 12288 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:30.057067017Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 117 117 117 F F F 1 020-039 69 69 69 107 107 107 T T T 2 040-059 74 74 74 117 117 117 F F F 3 060-079 72 72 72 112 112 112 F F F 4 080-099 70 70 70 109 109 109 F F F 5 100-119 74 74 74 116 116 116 F F F 6 120-139 74 74 74 117 117 117 F F F 7 140-159 74 74 74 117 117 117 F F F 8 160-179 74 74 74 117 117 117 T T T 9 180-199 74 74 74 117 117 117 T T T 10 200-219 73 73 73 115 115 115 F F F 11 220-239 73 73 73 115 115 115 F F F 12 240-259 72 72 72 112 112 112 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 74 74 74 117 117 117 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 74 74 74 117 117 117 F F F 21 420-439 73 73 73 115 115 115 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 72 72 72 112 112 112 F F F 24 480-499 71 71 71 110 110 110 F F F 25 500-519 72 72 72 114 114 114 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 74 74 74 117 117 117 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 73 73 73 115 115 115 F F F Max gen: 74, Max flush: 117 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 117 117 117 F F F 1 020-039 75 75 75 118 118 118 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 72 72 72 112 112 112 F F F 4 080-099 70 70 70 109 109 109 F F F 5 100-119 74 74 74 116 116 116 F F F 6 120-139 74 74 74 117 117 117 F F F 7 140-159 74 74 74 117 117 117 F F F 8 160-179 75 75 75 118 118 118 F F F 9 180-199 75 75 75 118 118 118 F F F 10 200-219 73 73 73 115 115 115 F F F 11 220-239 73 73 73 115 115 115 F F F 12 240-259 72 72 72 112 112 112 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 74 74 74 117 117 117 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 74 74 74 117 117 117 F F F 21 420-439 73 73 73 115 115 115 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 72 72 72 112 112 112 F F F 24 480-499 71 71 71 110 110 110 F F F 25 500-519 72 72 72 114 114 114 F F F 26 520-539 65 65 65 100 100 100 F F F 27 540-559 74 74 74 117 117 117 F F F 28 560-579 67 67 67 105 105 105 F F F 29 580-599 73 73 73 115 115 115 F F F Max gen: 75, Max flush: 118 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.285241988Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.329014899Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.329625895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.329661601Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.32967558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.329690508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496} {"msg":"Crucible 0c558a1f-ba27-4b49-9861-529de3309335 has session id: 7fdbacd8-88f9-4c0d-a1b0-f71347c09d53","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.330262008Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Upstairs opts: Upstairs UUID: 0c558a1f-ba27-4b49-9861-529de3309335, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.330300064Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Crucible stats registered with UUID: 0c558a1f-ba27-4b49-9861-529de3309335","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.330323392Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:34.330344039Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.330833848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.330866415Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496} {"msg":"0c558a1f-ba27-4b49-9861-529de3309335 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.330919459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.330946456Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"0","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.330992811Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"0","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331027198Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"1","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331057734Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"1","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331078162Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"2","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.33110069Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"2","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331123648Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331149635Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331173322Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"ds_connection connected from Ok(127.0.0.1:60477)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.33119944Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"ds_connection connected from Ok(127.0.0.1:42192)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331268102Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"ds_connection connected from Ok(127.0.0.1:59761)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331404258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331717765Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"1","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.331871549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"0","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.332347409Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"1","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.332488424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"0","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.332941106Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.333071573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.333211108Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"2","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.333714515Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"2","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[0]R flush_numbers[0..12]: [117, 118, 117, 112, 109, 116, 117, 117, 118, 118, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334211213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[0]R generation[0..12]: [74, 75, 74, 72, 70, 74, 74, 74, 75, 75, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334233991Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334251929Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[1]R flush_numbers[0..12]: [117, 118, 117, 112, 109, 116, 117, 117, 118, 118, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334268797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[1]R generation[0..12]: [74, 75, 74, 72, 70, 74, 74, 74, 75, 75, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334285375Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334302283Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[2]R flush_numbers[0..12]: [117, 118, 117, 112, 109, 116, 117, 117, 118, 118, 115, 115]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334320362Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[2]R generation[0..12]: [74, 75, 74, 72, 70, 74, 74, 74, 75, 75, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334337Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334353778Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:20:34.334369336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334392674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Next flush: 119","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334408272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334423781Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"":"downstairs","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334439519Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334454107Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"0","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334470136Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"1","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334485964Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"client":"2","":"downstairs","session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334501822Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334516801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"0c558a1f-ba27-4b49-9861-529de3309335 is now active with session: 7fdbacd8-88f9-4c0d-a1b0-f71347c09d53","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334536269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334550727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496,"session_id":"7fdbacd8-88f9-4c0d-a1b0-f71347c09d53"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334565386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496} {"msg":"Activated sub_volume 0c558a1f-ba27-4b49-9861-529de3309335","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:34.334579165Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 0 len 12288 data: 2 2 4 02/30 Flush 03/30 Write block 460 len 4096 data: 9 04/30 Write block 181 len 36864 data: 7 7 9 9 8 10 8 6 6 05/30 Read block 578 len 4096 06/30 Read block 208 len 20480 07/30 Write block 543 len 4096 data: 8 08/30 Flush 09/30 Write block 78 len 24576 data: 5 4 4 4 5 4 10/30 Write block 135 len 4096 data: 7 11/30 Write block 214 len 32768 data: 7 8 9 9 8 9 9 9 12/30 Read block 447 len 12288 13/30 Write block 419 len 32768 data: 9 6 4 5 3 3 5 6 14/30 Flush 15/30 Write block 142 len 8192 data: 11 11 16/30 Read block 396 len 28672 17/30 Read block 487 len 40960 18/30 Read block 133 len 32768 19/30 Read block 17 len 20480 20/30 Write block 561 len 32768 data: 8 7 5 5 5 6 4 4 21/30 Write block 517 len 24576 data: 9 8 6 5 4 4 22/30 Read block 251 len 40960 23/30 Write block 4 len 4096 data: 4 24/30 Read block 77 len 12288 25/30 Read block 529 len 8192 26/30 Write block 162 len 16384 data: 8 11 10 8 27/30 Write block 216 len 32768 data: 10 10 9 10 10 10 9 10 28/30 Read block 179 len 12288 29/30 Read block 29 len 16384 30/30 Read block 216 len 40960 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 2 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:34.354463036Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 119 119 119 T T T 1 020-039 75 75 75 118 118 118 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 76 76 76 121 121 121 F F F 4 080-099 76 76 76 121 121 121 F F F 5 100-119 74 74 74 116 116 116 F F F 6 120-139 76 76 76 121 121 121 F F F 7 140-159 74 74 74 117 117 117 T T T 8 160-179 75 75 75 118 118 118 T T T 9 180-199 76 76 76 120 120 120 F F F 10 200-219 76 76 76 121 121 121 T T T 11 220-239 76 76 76 121 121 121 T T T 12 240-259 72 72 72 112 112 112 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 74 74 74 117 117 117 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 76 76 76 121 121 121 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 76 76 76 120 120 120 F F F 24 480-499 71 71 71 110 110 110 F F F 25 500-519 72 72 72 114 114 114 T T T 26 520-539 65 65 65 100 100 100 T T T 27 540-559 76 76 76 120 120 120 F F F 28 560-579 67 67 67 105 105 105 T T T 29 580-599 73 73 73 115 115 115 F F F Max gen: 76, Max flush: 121 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 122 122 122 F F F 1 020-039 75 75 75 118 118 118 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 76 76 76 121 121 121 F F F 4 080-099 76 76 76 121 121 121 F F F 5 100-119 74 74 74 116 116 116 F F F 6 120-139 76 76 76 121 121 121 F F F 7 140-159 77 77 77 122 122 122 F F F 8 160-179 77 77 77 122 122 122 F F F 9 180-199 76 76 76 120 120 120 F F F 10 200-219 77 77 77 122 122 122 F F F 11 220-239 77 77 77 122 122 122 F F F 12 240-259 72 72 72 112 112 112 F F F 13 260-279 65 65 65 100 100 100 F F F 14 280-299 66 66 66 104 104 104 F F F 15 300-319 74 74 74 117 117 117 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 76 76 76 121 121 121 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 76 76 76 120 120 120 F F F 24 480-499 71 71 71 110 110 110 F F F 25 500-519 77 77 77 122 122 122 F F F 26 520-539 77 77 77 122 122 122 F F F 27 540-559 76 76 76 120 120 120 F F F 28 560-579 77 77 77 122 122 122 F F F 29 580-599 73 73 73 115 115 115 F F F Max gen: 77, Max flush: 122 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.610029268Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:38.652858129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.653250828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.684278268Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.684838239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.684871216Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.684885085Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.684905062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504} {"msg":"Crucible 5eea5918-7bf0-44e5-b93d-0596def9d844 has session id: 44d00a36-8604-46d5-9d4b-e905018de6bd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.68540719Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Upstairs opts: Upstairs UUID: 5eea5918-7bf0-44e5-b93d-0596def9d844, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.685438956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Crucible stats registered with UUID: 5eea5918-7bf0-44e5-b93d-0596def9d844","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.685454914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:38.685467783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.685849353Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.68587429Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504} {"msg":"5eea5918-7bf0-44e5-b93d-0596def9d844 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.685927095Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.685954062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"0","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.685983569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"0","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.685998048Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"1","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686013276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"1","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686027565Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"2","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686042513Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"2","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686056261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686076369Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686100257Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50888)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686121504Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:55001)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686139313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47308)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.68616381Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686754568Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"1","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686784105Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"2","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.686995593Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"0","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.687454344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"2","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.687531586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"1","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.687565813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"0","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.68806885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688094207Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[0]R flush_numbers[0..12]: [122, 118, 117, 121, 121, 116, 121, 122, 122, 120, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688108716Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[0]R generation[0..12]: [77, 75, 74, 76, 76, 74, 76, 77, 77, 76, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688127804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688142502Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[1]R flush_numbers[0..12]: [122, 118, 117, 121, 121, 116, 121, 122, 122, 120, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688156111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[1]R generation[0..12]: [77, 75, 74, 76, 76, 74, 76, 77, 77, 76, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.68816934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688182938Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[2]R flush_numbers[0..12]: [122, 118, 117, 121, 121, 116, 121, 122, 122, 120, 122, 122]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688196486Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[2]R generation[0..12]: [77, 75, 74, 76, 76, 74, 76, 77, 77, 76, 77, 77]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688209895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688223754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688236482Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688249201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.68826178Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688274568Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"":"downstairs","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688287247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid"Wait for a query_work_queue command to finish before sending IO :1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688307755Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"0","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688320953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"1","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688335512Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"client":"2","":"downstairs","session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.68834845Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688360419Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"5eea5918-7bf0-44e5-b93d-0596def9d844 is now active with session: 44d00a36-8604-46d5-9d4b-e905018de6bd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688372888Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688384597Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504,"session_id":"44d00a36-8604-46d5-9d4b-e905018de6bd"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688396516Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504} {"msg":"Activated sub_volume 5eea5918-7bf0-44e5-b93d-0596def9d844","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:38.688407674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 212 len 40960 data: 7 8 8 9 11 11 10 11 11 11 02/30 Write block 286 len 32768 data: 10 10 9 8 6 6 10 10 03/30 Write block 112 len 32768 data: 5 4 4 3 4 3 3 4 04/30 Write block 467 len 36864 data: 10 10 10 11 10 9 10 7 5 05/30 Read block 109 len 4096 06/30 Write block 515 len 32768 data: 10 8 10 9 7 6 5 5 07/30 Read block 116 len 20480 08/30 Write block 28 len 40960 data: 3 5 6 7 7 8 8 9 8 8 09/30 Write block 353 len 24576 data: 5 6 4 4 3 2 10/30 Write block 243 len 4096 data: 8 11/30 Write block 232 len 32768 data: 8 6 6 4 4 4 4 7 12/30 Write block 560 len 4096 data: 8 13/30 Read block 16 len 32768 14/30 Write block 315 len 4096 data: 8 15/30 Read block 75 len 24576 16/30 Read block 261 len 28672 17/30 Write block 136 len 8192 data: 8 8 18/30 Read block 321 len 36864 19/30 Read block 188 len 36864 20/30 Read block 53 len 16384 21/30 Read block 225 len 8192 22/30 Read block 257 len 36864 23/30 Read block 71 len 8192 24/30 Read block 104 len 40960 25/30 Write block 268 len 4096 data: 10 26/30 Read block 10 len 40960 27/30 Read block 254 len 28672 28/30 Read block 222 len 32768 29/30 Write block 219 len 28672 data: 12 12 12 10 11 8 7 30/30 Read block 71 len 28672 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Read 5 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 30 29 28 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:38.708225982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 122 122 122 F F F 1 020-039 75 75 75 118 118 118 T T T 2 040-059 74 74 74 117 117 117 F F F 3 060-079 76 76 76 121 121 121 F F F 4 080-099 76 76 76 121 121 121 F F F 5 100-119 74 74 74 116 116 116 T T T 6 120-139 76 76 76 121 121 121 T T T 7 140-159 77 77 77 122 122 122 F F F 8 160-179 77 77 77 122 122 122 F F F 9 180-199 76 76 76 120 120 120 F F F 10 200-219 77 77 77 122 122 122 T T T 11 220-239 77 77 77 122 122 122 T T T 12 240-259 72 72 72 112 112 112 T T T 13 260-279 65 65 65 100 100 100 T T T 14 280-299 66 66 66 104 104 104 T T T 15 300-319 74 74 74 117 117 117 T T T 16 320-339 65 65 65 100 100 100 F F F 17 340-359 70 70 70 108 108 108 T T T 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 76 76 76 121 121 121 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 76 76 76 120 120 120 T T T 24 480-499 71 71 71 110 110 110 F F F 25 500-519 77 77 77 122 122 122 T T T 26 520-539 77 77 77 122 122 122 T T T 27 540-559 76 76 76 120 120 120 F F F 28 560-579 77 77 77 122 122 122 T T T 29 580-599 73 73 73 115 115 115 F F F Max gen: 77, Max flush: 122 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 122 122 122 F F F 1 020-039 78 78 78 123 123 123 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 76 76 76 121 121 121 F F F 4 080-099 76 76 76 121 121 121 F F F 5 100-119 78 78 78 123 123 123 F F F 6 120-139 78 78 78 123 123 123 F F F 7 140-159 77 77 77 122 122 122 F F F 8 160-179 77 77 77 122 122 122 F F F 9 180-199 76 76 76 120 120 120 F F F 10 200-219 78 78 78 123 123 123 F F F 11 220-239 78 78 78 123 123 123 F F F 12 240-259 78 78 78 123 123 123 F F F 13 260-279 78 78 78 123 123 123 F F F 14 280-299 78 78 78 123 123 123 F F F 15 300-319 78 78 78 123 123 123 F F F 16 320-339 65 65 65 100 100 100 F F F 17 340-359 78 78 78 123 123 123 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 76 76 76 121 121 121 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 78 78 78 123 123 123 F F F 24 480-499 71 71 71 110 110 110 F F F 25 500-519 78 78 78 123 123 123 F F F 26 520-539 78 78 78 123 123 123 F F F 27 540-559 76 76 76 120 120 120 F F F 28 560-579 78 78 78 123 123 123 F F F 29 580-599 73 73 73 115 115 115 F F F Max gen: 78, Max flush: 123 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.018906414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:43.061633006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.062002817Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.093610176Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.094250219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.094283685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.094297754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.094312333Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512} {"msg":"Crucible 05b3b2a6-897b-4ab3-a7cc-698076be9223 has session id: 5c5edf69-4749-4546-bb38-6f2a29abe175","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.09491003Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Upstairs opts: Upstairs UUID: 05b3b2a6-897b-4ab3-a7cc-698076be9223, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.094943506Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Crucible stats registered with UUID: 05b3b2a6-897b-4ab3-a7cc-698076be9223","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.094964684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:43.094978432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095454933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.0954771Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512} {"msg":"05b3b2a6-897b-4ab3-a7cc-698076be9223 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095545163Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095561871Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"0","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095591988Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"0","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095616145Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"1","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095642863Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"1","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095718835Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"2","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095738153Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"2","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095758681Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095781858Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095807535Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"ds_connection connected from Ok(127.0.0.1:38794)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095838932Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"ds_connection connected from Ok(127.0.0.1:57919)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095867789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"ds_connection connected from Ok(127.0.0.1:45790)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.095894896Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.096554407Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"1","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.096583254Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"2","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.09681048Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"0","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.097330975Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"1","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.097360142Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"2","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.097637453Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"0","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.097938132Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098040001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[0]R flush_numbers[0..12]: [122, 123, 117, 121, 121, 123, 123, 122, 122, 120, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098212853Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[0]R generation[0..12]: [77, 78, 74, 76, 76, 78, 78, 77, 77, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098232901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098247509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[1]R flush_numbers[0..12]: [122, 123, 117, 121, 121, 123, 123, 122, 122, 120, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098261347Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[1]R generation[0..12]: [77, 78, 74, 76, 76, 78, 78, 77, 77, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098277636Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098291834Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[2]R flush_numbers[0..12]: [122, 123, 117, 121, 121, 123, 123, 122, 122, 120, 123, 123]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098305263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[2]R generation[0..12]: [77, 78, 74, 76, 76, 78, 78, 77, 77, 76, 78, 78]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098321111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098340699Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Max found gen is 79","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098354348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Generation requested: 80 >= found:79","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098367707Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098381225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098401753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"":"downstairs","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098414702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.09842648Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"0","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098439559Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"1","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098452538Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"client":"2","":"downstairs","session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098465726Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098479865Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"05b3b2a6-897b-4ab3-a7cc-698076be9223 is now active with session: 5c5edf69-4749-4546-bb38-6f2a29abe175","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098492803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098504582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512,"session_id":"5c5edf69-4749-4546-bb38-6f2a29abe175"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.098516521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512} {"msg":"Activated sub_volume 05b3b2a6-897b-4ab3-a7cc-698076be9223","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:43.09852747Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 333 len 40960 data: 4 4 6 7 7 7 6 6 7 5 02/30 Flush 03/30 Write block 149 len 20480 data: 6 6 5 6 7 04/30 Flush 05/30 Flush 06/30 Write block 130 len 20480 data: 6 6 8 9 9 07/30 Read block 591 len 36864 08/30 Read block 504 len 12288 09/30 Flush 10/30 Write block 61 len 8192 data: 7 9 11/30 Read block 41 len 8192 12/30 Read block 351 len 20480 13/30 Read block 341 len 40960 14/30 Read block 48 len 24576 15/30 Flush 16/30 Write block 586 len 36864 data: 9 9 8 7 5 5 5 4 4 17/30 Write block 471 len 40960 data: 11 10 11 8 6 5 4 4 3 3 18/30 Read block 450 len 12288 19/30 Write block 155 len 8192 data: 5 5 20/30 Write block 28 len 28672 data: 4 6 7 8 8 9 9 21/30 Read block 365 len 24576 22/30 Read block 317 len 12288 23/30 Write block 521 len 40960 data: 6 6 3 3 3 5 5 5 3 3 24/30 Read block 151 len 8192 25/30 Write block 224 len 24576 data: 9 8 5 5 4 5 26/30 Write block 20 len 20480 data: 6 6 5 5 3 27/30 Read block 361 len 36864 28/30 Write block 314 len 12288 data: 9 9 5 29/30 Read block 19 len 24576 30/30 Read block 185 len 20480 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:43.119795185Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 122 122 122 F F F 1 020-039 78 78 78 123 123 123 T T T 2 040-059 74 74 74 117 117 117 F F F 3 060-079 80 80 80 128 128 128 F F F 4 080-099 76 76 76 121 121 121 F F F 5 100-119 78 78 78 123 123 123 F F F 6 120-139 80 80 80 127 127 127 F F F 7 140-159 80 80 80 125 125 125 T T T 8 160-179 77 77 77 122 122 122 F F F 9 180-199 76 76 76 120 120 120 F F F 10 200-219 78 78 78 123 123 123 F F F 11 220-239 78 78 78 123 123 123 T T T 12 240-259 78 78 78 123 123 123 F F F 13 260-279 78 78 78 123 123 123 F F F 14 280-299 78 78 78 123 123 123 F F F 15 300-319 78 78 78 123 123 123 T T T 16 320-339 80 80 80 124 124 124 F F F 17 340-359 80 80 80 124 124 124 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 76 76 76 121 121 121 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 78 78 78 123 123 123 T T T 24 480-499 71 71 71 110 110 110 T T T 25 500-519 78 78 78 123 123 123 F F F 26 520-539 78 78 78 123 123 123 T T T 27 540-559 76 76 76 120 120 120 F F F 28 560-579 78 78 78 123 123 123 F F F 29 580-599 73 73 73 115 115 115 T T T Max gen: 80, Max flush: 128 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 77 77 77 122 122 122 F F F 1 020-039 81 81 81 129 129 129 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 80 80 80 128 128 128 F F F 4 080-099 76 76 76 121 121 121 F F F 5 100-119 78 78 78 123 123 123 F F F 6 120-139 80 80 80 127 127 127 F F F 7 140-159 81 81 81 129 129 129 F F F 8 160-179 77 77 77 122 122 122 F F F 9 180-199 76 76 76 120 120 120 F F F 10 200-219 78 78 78 123 123 123 F F F 11 220-239 81 81 81 129 129 129 F F F 12 240-259 78 78 78 123 123 123 F F F 13 260-279 78 78 78 123 123 123 F F F 14 280-299 78 78 78 123 123 123 F F F 15 300-319 81 81 81 129 129 129 F F F 16 320-339 80 80 80 124 124 124 F F F 17 340-359 80 80 80 124 124 124 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 76 76 76 121 121 121 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 81 81 81 129 129 129 F F F 24 480-499 81 81 81 129 129 129 F F F 25 500-519 78 78 78 123 123 123 F F F 26 520-539 81 81 81 129 129 129 F F F 27 540-559 76 76 76 120 120 120 F F F 28 560-579 78 78 78 123 123 123 F F F 29 580-599 81 81 81 129 129 129 F F F Max gen: 81, Max flush: 129 Begin loop 41 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.403438787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:47.446088627Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.446462678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.477672639Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.47823071Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.478263317Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.478276736Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.478291204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520} {"msg":"Crucible 7dc2f7d0-8f73-4ce2-856a-ff1b02b0f4f9 has session id: 5fda856b-57c7-4c76-9433-2299db75f735","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.478924168Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Upstairs opts: Upstairs UUID: 7dc2f7d0-8f73-4ce2-856a-ff1b02b0f4f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.478963623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Crucible stats registered with UUID: 7dc2f7d0-8f73-4ce2-856a-ff1b02b0f4f9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.478986471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:47.47900022Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479371621Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479394678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520} {"msg":"7dc2f7d0-8f73-4ce2-856a-ff1b02b0f4f9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479462111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479479829Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"0","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479510906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"0","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479525564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"1","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479546262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"1","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.47957001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"2","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479607676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"2","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479631473Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.47965858Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479684937Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"ds_connection connected from Ok(127.0.0.1:62840)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479710835Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"ds_connection connected from Ok(127.0.0.1:45492)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479734842Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"ds_connection connected from Ok(127.0.0.1:51059)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.479916953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.480481744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"1","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.48052074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"2","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.480625959Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"0","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481094269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"2","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481241704Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"1","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481349963Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"0","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481628133Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481796936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[0]R flush_numbers[0..12]: [122, 129, 117, 128, 121, 123, 127, 129, 122, 120, 123, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481933291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[0]R generation[0..12]: [77, 81, 74, 80, 76, 78, 80, 81, 77, 76, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481960049Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481975777Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[1]R flush_numbers[0..12]: [122, 129, 117, 128, 121, 123, 127, 129, 122, 120, 123, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.481989926Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[1]R generation[0..12]: [77, 81, 74, 80, 76, 78, 80, 81, 77, 76, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482018363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482032931Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[2]R flush_numbers[0..12]: [122, 129, 117, 128, 121, 123, 127, 129, 122, 120, 123, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482046469Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[2]R generation[0..12]: [77, 81, 74, 80, 76, 78, 80, 81, 77, 76, 78, 81]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482059608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482073237Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Max found gen is 82","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-03-18T15:20:47.482097324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482111223Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482124231Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.48213729Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"":"downstairs","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482150269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482161997Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"0","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482174856Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"1","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482187395Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"client":"2","":"downstairs","session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482200223Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482212172Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"7dc2f7d0-8f73-4ce2-856a-ff1b02b0f4f9 is now active with session: 5fda856b-57c7-4c76-9433-2299db75f735","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482224681Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:20:47.48223646Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520,"session_id":"5fda856b-57c7-4c76-9433-2299db75f735"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482248488Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520} {"msg":"Activated sub_volume 7dc2f7d0-8f73-4ce2-856a-ff1b02b0f4f9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:47.482259327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 405 len 8192 data: 11 10 02/30 Write block 147 len 24576 data: 8 9 7 7 6 7 03/30 Write block 217 len 16384 data: 12 11 13 13 04/30 Write block 535 len 36864 data: 7 7 8 8 7 5 4 6 9 05/30 Write block 524 len 12288 data: 4 4 6 06/30 Read block 360 len 8192 07/30 Write block 91 len 40960 data: 5 5 6 5 7 8 7 7 10 11 08/30 Read block 341 len 8192 09/30 Read block 319 len 40960 10/30 Read block 5 len 24576 11/30 Write block 338 len 32768 data: 8 7 7 8 6 4 4 4 12/30 Write block 415 len 12288 data: 7 9 10 13/30 Write block 162 len 28672 data: 9 12 11 9 9 10 9 14/30 Write block 0 len 28672 data: 3 3 5 4 5 4 6 15/30 Read block 454 len 40960 16/30 Write block 70 len 16384 data: 8 7 7 6 17/30 Write block 480 len 40960 data: 4 7 7 6 7 8 8 8 8 6 18/30 Flush 19/30 Write block 509 len 8192 data: 7 9 20/30 Write block 73 len 24576 data: 7 7 6 6 8 6 21/30 Read block 306 len 4096 22/30 Write block 31 len 8192 data: 9 9 23/30 Read block 591 len 36864 24/30 Read block 85 len 40960 25/30 Read block 455 len 4096 26/30 Read block 186 len 12288 27/30 Read block 585 len 8192 28/30 Read block 453 len 36864 29/30 Write block 468 len 12288 data: 11 11 12 30/30 Read block 324 len 12288 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 2 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 12 11 10 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:47.501628042Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 130 130 130 F F F 1 020-039 81 81 81 129 129 129 T T T 2 040-059 74 74 74 117 117 117 F F F 3 060-079 82 82 82 130 130 130 T T T 4 080-099 82 82 82 130 130 130 F F F 5 100-119 82 82 82 130 130 130 F F F 6 120-139 80 80 80 127 127 127 F F F 7 140-159 82 82 82 130 130 130 F F F 8 160-179 82 82 82 130 130 130 F F F 9 180-199 76 76 76 120 120 120 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 78 78 78 123 123 123 F F F 13 260-279 78 78 78 123 123 123 F F F 14 280-299 78 78 78 123 123 123 F F F 15 300-319 81 81 81 129 129 129 F F F 16 320-339 82 82 82 130 130 130 F F F 17 340-359 82 82 82 130 130 130 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 82 82 82 130 130 130 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 81 81 81 129 129 129 T T T 24 480-499 82 82 82 130 130 130 F F F 25 500-519 78 78 78 123 123 123 T T T 26 520-539 82 82 82 130 130 130 F F F 27 540-559 82 82 82 130 130 130 F F F 28 560-579 78 78 78 123 123 123 F F F 29 580-599 81 81 81 129 129 129 F F F Max gen: 82, Max flush: 130 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 130 130 130 F F F 1 020-039 83 83 83 131 131 131 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 83 83 83 131 131 131 F F F 4 080-099 82 82 82 130 130 130 F F F 5 100-119 82 82 82 130 130 130 F F F 6 120-139 80 80 80 127 127 127 F F F 7 140-159 82 82 82 130 130 130 F F F 8 160-179 82 82 82 130 130 130 F F F 9 180-199 76 76 76 120 120 120 F F F 10 200-219 82 82 82 130 130 130 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 78 78 78 123 123 123 F F F 13 260-279 78 78 78 123 123 123 F F F 14 280-299 78 78 78 123 123 123 F F F 15 300-319 81 81 81 129 129 129 F F F 16 320-339 82 82 82 130 130 130 F F F 17 340-359 82 82 82 130 130 130 F F F 18 360-379 72 72 72 113 113 113 F F F 19 380-399 72 72 72 113 113 113 F F F 20 400-419 82 82 82 130 130 130 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 F F F 23 460-479 83 83 83 131 131 131 F F F 24 480-499 82 82 82 130 130 130 F F F 25 500-519 83 83 83 131 131 131 F F F 26 520-539 82 82 82 130 130 130 F F F 27 540-559 82 82 82 130 130 130 F F F 28 560-579 78 78 78 123 123 123 F F F 29 580-599 81 81 81 129 129 129 F F F Max gen: 83, Max flush: 131 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.76686174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.81321499Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.813903428Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.813940434Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.813956842Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.81397398Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528} {"msg":"Crucible dfde705d-a0c3-4e59-9cee-609c8d00b935 has session id: e50287d0-084e-4b1d-a12e-53b4cca6d7ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.814463319Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Upstairs opts: Upstairs UUID: dfde705d-a0c3-4e59-9cee-609c8d00b935, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.814566618Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Crucible stats registered with UUID: dfde705d-a0c3-4e59-9cee-609c8d00b935","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.814586846Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:51.814599705Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815109011Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815133378Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528} {"msg":"dfde705d-a0c3-4e59-9cee-609c8d00b935 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815220499Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815237578Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"0","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815265935Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"0","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815279973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"1","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815294262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"1","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.81531134Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815336987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815362325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815382812Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815412589Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:51.815445336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:51.815474063Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:63698)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.81550081Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:20:51.815529907Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:50315)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815655074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.815733015Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.816139813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"0","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.81616785Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"1","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.816759748Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"1","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.816869486Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"0","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.817290112Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:20:51.817436107Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.816567899Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:52610)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.816715683Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.817850184Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.818470669Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[0]R flush_numbers[0..12]: [130, 131, 117, 131, 130, 130, 127, 130, 130, 120, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819052948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[0]R generation[0..12]: [82, 83, 74, 83, 82, 82, 80, 82, 82, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819075045Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819090384Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[1]R flush_numbers[0..12]: [130, 131, 117, 131, 130, 130, 127, 130, 130, 120, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819104172Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[1]R generation[0..12]: [82, 83, 74, 83, 82, 82, 80, 82, 82, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819117311Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.81913088Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[2]R flush_numbers[0..12]: [130, 131, 117, 131, 130, 130, 127, 130, 130, 120, 130, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819144498Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[2]R generation[0..12]: [82, 83, 74, 83, 82, 82, 80, 82, 82, 76, 82, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819157977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819171715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819187663Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819210781Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819240028Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819263715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"":"downstairs","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819285643Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819305311Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"0","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2025-03-18T15:21:01.819329969Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"1","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819357736Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"client":"2","":"downstairs","session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819372764Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819384823Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"dfde705d-a0c3-4e59-9cee-609c8d00b935 is now active with session: e50287d0-084e-4b1d-a12e-53b4cca6d7ea","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819397512Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.81940924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528,"session_id":"e50287d0-084e-4b1d-a12e-53b4cca6d7ea"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819421059Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528} {"msg":"Activated sub_volume dfde705d-a0c3-4e59-9cee-609c8d00b935","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:01.819431968Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 350 len 40960 data: 5 5 5 6 7 5 5 4 3 4 03/30 Read block 213 len 28672 04/30 Write block 558 len 4096 data: 7 05/30 Write block 137 len 40960 data: 9 9 11 10 10 12 12 11 9 9 06/30 Write block 211 len 16384 data: 7 8 9 9 07/30 Write block 303 len 12288 data: 6 6 4 08/30 Read block 441 len 32768 09/30 Write block 110 len 40960 data: 6 6 6 5 5 4 5 4 4 5 10/30 Write block 269 len 16384 data: 9 9 9 10 11/30 Read block 240 len 24576 12/30 Read block 306 len 20480 13/30 Read block 13 len 40960 14/30 Read block 451 len 32768 15/30 Read block 408 len 16384 16/30 Write block 440 len 32768 data: 5 4 4 6 6 6 8 8 17/30 Read block 465 len 12288 18/30 Read block 510 len 4096 19/30 Read block 92 len 8192 20/30 Read block 507 len 20480 21/30 Read block 364 len 20480 22/30 Write block 395 len 20480 data: 6 8 8 9 10 23/30 Write block 391 len 40960 data: 6 6 7 6 7 9 9 10 11 10 24/30 Write block 466 len 32768 data: 10 11 12 12 13 12 11 12 25/30 Read block 371 len 28672 26/30 Write block 197 len 40960 data: 6 5 5 8 8 7 7 4 5 6 27/30 Write block 374 len 32768 data: 7 6 8 8 8 8 7 7 28/30 Read block 591 len 4096 29/30 Write block 499 len 36864 data: 3 5 5 6 6 5 6 5 5 30/30 Read block 342 len 36864 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 10 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 28 29 28 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:01.841465523Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 130 130 130 F F F 1 020-039 83 83 83 131 131 131 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 83 83 83 131 131 131 F F F 4 080-099 82 82 82 130 130 130 F F F 5 100-119 82 82 82 130 130 130 T T T 6 120-139 80 80 80 127 127 127 T T T 7 140-159 82 82 82 130 130 130 T T T 8 160-179 82 82 82 130 130 130 F F F 9 180-199 76 76 76 120 120 120 T T T 10 200-219 82 82 82 130 130 130 T T T 11 220-239 82 82 82 130 130 130 F F F 12 240-259 78 78 78 123 123 123 F F F 13 260-279 78 78 78 123 123 123 T T T 14 280-299 78 78 78 123 123 123 F F F 15 300-319 81 81 81 129 129 129 T T T 16 320-339 82 82 82 130 130 130 F F F 17 340-359 82 82 82 130 130 130 T T T 18 360-379 72 72 72 113 113 113 T T T 19 380-399 72 72 72 113 113 113 T T T 20 400-419 82 82 82 130 130 130 T T T 21 420-439 76 76 76 121 121 121 F F F 22 440-459 70 70 70 109 109 109 T T T 23 460-479 83 83 83 131 131 131 T T T 24 480-499 82 82 82 130 130 130 T T T 25 500-519 83 83 83 131 131 131 T T T 26 520-539 82 82 82 130 130 130 F F F 27 540-559 82 82 82 130 130 130 T T T 28 560-579 78 78 78 123 123 123 F F F 29 580-599 81 81 81 129 129 129 F F F Max gen: 83, Max flush: 131 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 130 130 130 F F F 1 020-039 83 83 83 131 131 131 F F F 2 040-059 74 74 74 117 117 117 F F F 3 060-079 83 83 83 131 131 131 F F F 4 080-099 82 82 82 130 130 130 F F F 5 100-119 84 84 84 132 132 132 F F F 6 120-139 84 84 84 132 132 132 F F F 7 140-159 84 84 84 132 132 132 F F F 8 160-179 82 82 82 130 130 130 F F F 9 180-199 84 84 84 132 132 132 F F F 10 200-219 84 84 84 132 132 132 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 78 78 78 123 123 123 F F F 13 260-279 84 84 84 132 132 132 F F F 14 280-299 78 78 78 123 123 123 F F F 15 300-319 84 84 84 132 132 132 F F F 16 320-339 82 82 82 130 130 130 F F F 17 340-359 84 84 84 132 132 132 F F F 18 360-379 84 84 84 132 132 132 F F F 19 380-399 84 84 84 132 132 132 F F F 20 400-419 84 84 84 132 132 132 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 84 84 84 132 132 132 F F F 23 460-479 84 84 84 132 132 132 F F F 24 480-499 84 84 84 132 132 132 F F F 25 500-519 84 84 84 132 132 132 F F F 26 520-539 82 82 82 130 130 130 F F F 27 540-559 84 84 84 132 132 132 F F F 28 560-579 78 78 78 123 123 123 F F F 29 580-599 81 81 81 129 129 129 F F F Max gen: 84, Max flush: 132 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.139515656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:16.181755068Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.182106751Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.212849122Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.213384145Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.213424011Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.213445409Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.213464597Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536} {"msg":"Crucible 9f61fc75-af0a-467d-bb87-ee606e263fa1 has session id: a69db8da-91a1-4512-872e-70bb43563b4e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214083162Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Upstairs opts: Upstairs UUID: 9f61fc75-af0a-467d-bb87-ee606e263fa1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214110489Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Crucible stats registered with UUID: 9f61fc75-af0a-467d-bb87-ee606e263fa1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214125358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:16.214137666Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214522256Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214545394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536} {"msg":"9f61fc75-af0a-467d-bb87-ee606e263fa1 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214607087Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214624715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"0","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214652082Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"0","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214665781Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"1","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.21467998Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"1","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214702567Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"2","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214804466Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"2","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214832524Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214857561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214881228Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60427)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.214907225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:58557)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.215030203Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46635)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.21505913Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.215555428Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"1","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.215592344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"2","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.215749507Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"0","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.216242405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"1","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.216282361Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"2","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.216312918Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"0","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.216812735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.216925333Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[0]R flush_numbers[0..12]: [130, 131, 117, 131, 130, 132, 132, 132, 130, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.216998276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[0]R generation[0..12]: [82, 83, 74, 83, 82, 84, 84, 84, 82, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217018394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217033102Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[1]R flush_numbers[0..12]: [130, 131, 117, 131, 130, 132, 132, 132, 130, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217046641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[1]R generation[0..12]: [82, 83, 74, 83, 82, 84, 84, 84, 82, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217060189Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217073758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[2]R flush_numbers[0..12]: [130, 131, 117, 131, 130, 132, 132, 132, 130, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217087156Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[2]R generation[0..12]: [82, 83, 74, 83, 82, 84, 84, 84, 82, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217100105Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217113414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Max found gen is 85","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217125863Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Generation requested: 86 >= found:85","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217138461Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.21715086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217163379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"":"downstairs","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217176237Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:21:16.217191036Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"0","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217209624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"1","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217222502Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"client":"2","":"downstairs","session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217235171Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.21724719Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"9f61fc75-af0a-467d-bb87-ee606e263fa1 is now active with session: a69db8da-91a1-4512-872e-70bb43563b4e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217259618Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217271087Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536,"session_id":"a69db8da-91a1-4512-872e-70bb43563b4e"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217283006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536} {"msg":"Activated sub_volume 9f61fc75-af0a-467d-bb87-ee606e263fa1","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:16.217293725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 584 len 32768 data: 11 9 10 10 9 8 6 6 02/30 Write block 504 len 36864 data: 6 7 6 6 5 8 10 11 10 03/30 Flush 04/30 Write block 47 len 40960 data: 9 7 8 8 6 7 6 8 8 9 05/30 Read block 304 len 24576 06/30 Write block 141 len 4096 data: 11 07/30 Write block 66 len 16384 data: 9 8 7 8 08/30 Write block 255 len 16384 data: 8 5 4 4 09/30 Read block 563 len 8192 10/30 Write block 37 len 20480 data: 9 7 5 6 9 11/30 Read block 402 len 24576 12/30 Read block 512 len 20480 13/30 Write block 523 len 4096 data: 4 14/30 Write block 323 len 32768 data: 6 8 5 5 4 4 4 6 15/30 Write block 327 len 36864 data: 5 5 5 7 6 5 5 5 7 16/30 Write block 525 len 40960 data: 5 7 6 6 4 4 3 4 5 7 17/30 Flush 18/30 Write block 313 len 36864 data: 7 10 10 6 6 6 7 5 7 19/30 Write block 493 len 8192 data: 6 6 20/30 Read block 478 len 40960 21/30 Read block 494 len 32768 22/30 Flush 23/30 Write block 587 len 4096 data: 11 24/30 Flush 25/30 Write block 466 len 36864 data: 11 12 13 13 14 13 12 13 9 26/30 Read block 433 len 24576 27/30 Write block 99 len 24576 data: 11 12 11 11 10 9 28/30 Write block 541 len 4096 data: 5 29/30 Write block 471 len 4096 data: 14 30/30 Write block 571 len 36864 data: 4 6 7 5 5 4 4 5 4 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 1 Sent Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:16.23685664Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 130 130 130 F F F 1 020-039 86 86 86 134 134 134 F F F 2 040-059 86 86 86 134 134 134 F F F 3 060-079 86 86 86 134 134 134 F F F 4 080-099 82 82 82 130 130 130 T T T 5 100-119 84 84 84 132 132 132 T T T 6 120-139 84 84 84 132 132 132 F F F 7 140-159 86 86 86 134 134 134 F F F 8 160-179 82 82 82 130 130 130 F F F 9 180-199 84 84 84 132 132 132 F F F 10 200-219 84 84 84 132 132 132 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 86 86 86 134 134 134 F F F 13 260-279 84 84 84 132 132 132 F F F 14 280-299 78 78 78 123 123 123 F F F 15 300-319 86 86 86 135 135 135 F F F 16 320-339 86 86 86 135 135 135 F F F 17 340-359 84 84 84 132 132 132 F F F 18 360-379 84 84 84 132 132 132 F F F 19 380-399 84 84 84 132 132 132 F F F 20 400-419 84 84 84 132 132 132 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 84 84 84 132 132 132 F F F 23 460-479 84 84 84 132 132 132 T T T 24 480-499 86 86 86 135 135 135 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 86 86 86 134 134 134 F F F 27 540-559 84 84 84 132 132 132 T T T 28 560-579 78 78 78 123 123 123 T T T 29 580-599 86 86 86 136 136 136 F F F Max gen: 86, Max flush: 136 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 130 130 130 F F F 1 020-039 86 86 86 134 134 134 F F F 2 040-059 86 86 86 134 134 134 F F F 3 060-079 86 86 86 134 134 134 F F F 4 080-099 87 87 87 137 137 137 F F F 5 100-119 87 87 87 137 137 137 F F F 6 120-139 84 84 84 132 132 132 F F F 7 140-159 86 86 86 134 134 134 F F F 8 160-179 82 82 82 130 130 130 F F F 9 180-199 84 84 84 132 132 132 F F F 10 200-219 84 84 84 132 132 132 F F F 11 220-239 82 82 82 130 130 130 F F F 12 240-259 86 86 86 134 134 134 F F F 13 260-279 84 84 84 132 132 132 F F F 14 280-299 78 78 78 123 123 123 F F F 15 300-319 86 86 86 135 135 135 F F F 16 320-339 86 86 86 135 135 135 F F F 17 340-359 84 84 84 132 132 132 F F F 18 360-379 84 84 84 132 132 132 F F F 19 380-399 84 84 84 132 132 132 F F F 20 400-419 84 84 84 132 132 132 F F F 21 420-439 76 76 76 121 121 121 F F F 22 440-459 84 84 84 132 132 132 F F F 23 460-479 87 87 87 137 137 137 F F F 24 480-499 86 86 86 135 135 135 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 86 86 86 134 134 134 F F F 27 540-559 87 87 87 137 137 137 F F F 28 560-579 87 87 87 137 137 137 F F F 29 580-599 86 86 86 136 136 136 F F F Max gen: 87, Max flush: 137 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.504002367Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.547766989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.548321201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.548356697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.548371056Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.548385754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544} {"msg":"Crucible 58d7add8-2584-47b7-9f64-6483c1f23457 has session id: 88762013-0302-4d68-abef-98579d9635e5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.54890823Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Upstairs opts: Upstairs UUID: 58d7add8-2584-47b7-9f64-6483c1f23457, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.548937416Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Crucible stats registered with UUID: 58d7add8-2584-47b7-9f64-6483c1f23457","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.548952685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:20.548965213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549503577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549532754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544} {"msg":"58d7add8-2584-47b7-9f64-6483c1f23457 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549601497Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549628404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"0","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549672439Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"0","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549700486Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"1","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549725564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"1","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549749521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"2","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549773068Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"2","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549794496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549817684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549841701Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46170)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549867189Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41364)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549892746Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58279)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.549918453Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.550463206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"1","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.550644677Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"0","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.550963123Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"1","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.55109492Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"0","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.551614995Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.551700596Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.55213825Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"2","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.552634768Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"2","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[0]R flush_numbers[0..12]: [130, 134, 134, 134, 137, 137, 132, 134, 130, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553142055Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[0]R generation[0..12]: [82, 86, 86, 86, 87, 87, 84, 86, 82, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553166442Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553181461Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[1]R flush_numbers[0..12]: [130, 134, 134, 134, 137, 137, 132, 134, 130, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553195339Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[1]R generation[0..12]: [82, 86, 86, 86, 87, 87, 84, 86, 82, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553208518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553222076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[2]R flush_numbers[0..12]: [130, 134, 134, 134, 137, 137, 132, 134, 130, 132, 132, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553235345Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[2]R generation[0..12]: [82, 86, 86, 86, 87, 87, 84, 86, 82, 84, 84, 82]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553251873Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553266062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.55327912Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:21:20.553292109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553311547Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553324006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"":"downstairs","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553336994Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553348763Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"0","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553361691Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"1","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.55337421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"client":"2","":"downstairs","session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553387069Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553398968Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"58d7add8-2584-47b7-9f64-6483c1f23457 is now active with session: 88762013-0302-4d68-abef-98579d9635e5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553411476Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553423025Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544,"session_id":"88762013-0302-4d68-abef-98579d9635e5"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553434784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544} {"msg":"Activated sub_volume 58d7add8-2584-47b7-9f64-6483c1f23457","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:20.553445913Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 160 len 8192 data: 6 6 02/30 Read block 378 len 28672 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Write block 9 len 16384 data: 6 6 7 9 07/30 Write block 234 len 40960 data: 7 5 5 5 5 8 8 7 7 9 08/30 Read block 236 len 16384 09/30 Write block 96 len 32768 data: 9 8 8 12 13 12 12 11 10/30 Read block 180 len 20480 11/30 Write block 310 len 20480 data: 5 7 7 8 11 12/30 Write block 326 len 32768 data: 6 6 6 6 8 7 6 6 13/30 Read block 165 len 20480 14/30 Write block 427 len 40960 data: 7 8 11 13 12 11 10 9 9 6 15/30 Write block 300 len 32768 data: 8 7 6 7 7 5 6 8 16/30 Read block 258 len 20480 17/30 Write block 555 len 4096 data: 9 18/30 Read block 312 len 28672 19/30 Write block 429 len 36864 data: 12 14 13 12 11 10 10 7 6 20/30 Read block 432 len 8192 21/30 Write block 574 len 40960 data: 6 6 5 5 6 5 4 8 8 8 22/30 Write block 529 len 28672 data: 5 5 4 5 6 8 8 23/30 Read block 319 len 20480 24/30 Write block 280 len 28672 data: 7 8 10 12 13 12 11 25/30 Read block 64 len 28672 26/30 Write block 372 len 36864 data: 8 8 8 7 9 9 9 9 8 27/30 Write block 30 len 4096 data: 8 28/30 Write block 479 len 32768 data: 4 5 8 8 7 8 9 9 29/30 Read block 392 len 20480 30/30 Write block 435 len 40960 data: 11 8 7 7 5 6 5 5 7 7 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 4 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 8 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:20.57441367Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 130 130 130 T T T 1 020-039 86 86 86 134 134 134 T T T 2 040-059 86 86 86 134 134 134 F F F 3 060-079 86 86 86 134 134 134 F F F 4 080-099 87 87 87 137 137 137 T T T 5 100-119 87 87 87 137 137 137 T T T 6 120-139 84 84 84 132 132 132 F F F 7 140-159 86 86 86 134 134 134 F F F 8 160-179 88 88 88 138 138 138 F F F 9 180-199 84 84 84 132 132 132 F F F 10 200-219 84 84 84 132 132 132 F F F 11 220-239 82 82 82 130 130 130 T T T 12 240-259 86 86 86 134 134 134 T T T 13 260-279 84 84 84 132 132 132 F F F 14 280-299 78 78 78 123 123 123 T T T 15 300-319 86 86 86 135 135 135 T T T 16 320-339 86 86 86 135 135 135 T T T 17 340-359 84 84 84 132 132 132 F F F 18 360-379 84 84 84 132 132 132 T T T 19 380-399 84 84 84 132 132 132 T T T 20 400-419 84 84 84 132 132 132 F F F 21 420-439 76 76 76 121 121 121 T T T 22 440-459 84 84 84 132 132 132 T T T 23 460-479 87 87 87 137 137 137 T T T 24 480-499 86 86 86 135 135 135 T T T 25 500-519 86 86 86 133 133 133 F F F 26 520-539 86 86 86 134 134 134 T T T 27 540-559 87 87 87 137 137 137 T T T 28 560-579 87 87 87 137 137 137 T T T 29 580-599 86 86 86 136 136 136 T T T Max gen: 88, Max flush: 138 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 139 139 139 F F F 1 020-039 89 89 89 139 139 139 F F F 2 040-059 86 86 86 134 134 134 F F F 3 060-079 86 86 86 134 134 134 F F F 4 080-099 89 89 89 139 139 139 F F F 5 100-119 89 89 89 139 139 139 F F F 6 120-139 84 84 84 132 132 132 F F F 7 140-159 86 86 86 134 134 134 F F F 8 160-179 88 88 88 138 138 138 F F F 9 180-199 84 84 84 132 132 132 F F F 10 200-219 84 84 84 132 132 132 F F F 11 220-239 89 89 89 139 139 139 F F F 12 240-259 89 89 89 139 139 139 F F F 13 260-279 84 84 84 132 132 132 F F F 14 280-299 89 89 89 139 139 139 F F F 15 300-319 89 89 89 139 139 139 F F F 16 320-339 89 89 89 139 139 139 F F F 17 340-359 84 84 84 132 132 132 F F F 18 360-379 89 89 89 139 139 139 F F F 19 380-399 89 89 89 139 139 139 F F F 20 400-419 84 84 84 132 132 132 F F F 21 420-439 89 89 89 139 139 139 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 89 89 89 139 139 139 F F F 24 480-499 89 89 89 139 139 139 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 89 89 89 139 139 139 F F F 27 540-559 89 89 89 139 139 139 F F F 28 560-579 89 89 89 139 139 139 F F F 29 580-599 89 89 89 139 139 139 F F F Max gen: 89, Max flush: 139 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.881470265Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.92645007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.927479242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.927518078Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.927533446Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.927548285Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552} {"msg":"Crucible 066f72c4-9ffc-4639-a226-ab7606328d6d has session id: fe49cea9-f831-4735-8ce7-57ad072f1699","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928086688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Upstairs opts: Upstairs UUID: 066f72c4-9ffc-4639-a226-ab7606328d6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.92816408Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Crucible stats registered with UUID: 066f72c4-9ffc-4639-a226-ab7606328d6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928200966Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:34.928222604Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928621892Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928654068Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552} {"msg":"066f72c4-9ffc-4639-a226-ab7606328d6d active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928707853Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.92873175Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"0","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928773906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"0","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928805782Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"1","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928838149Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"1","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928868616Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"2","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928898373Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"2","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.92892647Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928954167Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.928984383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"ds_connection connected from Ok(127.0.0.1:47139)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.92901468Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"ds_connection connected from Ok(127.0.0.1:65499)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.929159035Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"ds_connection connected from Ok(127.0.0.1:56207)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.929198521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.929669332Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"2","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.929795808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"0","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.930270688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"2","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.93035358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"0","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.930815581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"1","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.930896713Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.930963876Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.931372363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"1","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[0]R flush_numbers[0..12]: [139, 139, 134, 134, 139, 139, 132, 134, 138, 132, 132, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.93187823Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[0]R generation[0..12]: [89, 89, 86, 86, 89, 89, 84, 86, 88, 84, 84, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.931896888Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.931911086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[1]R flush_numbers[0..12]: [139, 139, 134, 134, 139, 139, 132, 134, 138, 132, 132, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.931925705Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[1]R generation[0..12]: [89, 89, 86, 86, 89, 89, 84, 86, 88, 84, 84, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.931938903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.931952772Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[2]R flush_numbers[0..12]: [139, 139, 134, 134, 139, 139, 132, 134, 138, 132, 132, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.93196679Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[2]R generation[0..12]: [89, 89, 86, 86, 89, 89, 84, 86, 88, 84, 84, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.931981039Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.931994627Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932007136Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932020635Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932033193Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932048702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"":"downstairs","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.93206242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932074149Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"0","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932087138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"1","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932100056Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"client":"2","":"downstairs","session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932112915Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Wait for a query_work_queue command to finish before sending IO Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932131073Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"066f72c4-9ffc-4639-a226-ab7606328d6d is now active with session: fe49cea9-f831-4735-8ce7-57ad072f1699","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932143612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932155281Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552,"session_id":"fe49cea9-f831-4735-8ce7-57ad072f1699"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932167159Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552} {"msg":"Activated sub_volume 066f72c4-9ffc-4639-a226-ab7606328d6d","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:34.932178038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 550 len 24576 data: 9 10 11 11 9 10 02/30 Read block 187 len 40960 03/30 Read block 422 len 20480 04/30 Read block 320 len 24576 05/30 Write block 557 len 40960 data: 8 8 5 9 9 8 6 6 6 7 06/30 Write block 18 len 8192 data: 7 7 07/30 Write block 394 len 12288 data: 7 8 10 08/30 Read block 119 len 32768 09/30 Write block 158 len 24576 data: 6 6 7 7 10 13 10/30 Flush 11/30 Write block 542 len 4096 data: 7 12/30 Read block 25 len 32768 13/30 Write block 51 len 40960 data: 7 8 7 9 9 10 10 9 9 8 14/30 Write block 403 len 16384 data: 11 12 12 11 15/30 Read block 499 len 4096 16/30 Read block 361 len 24576 17/30 Read block 560 len 4096 18/30 Write block 191 len 40960 data: 9 9 9 8 7 9 7 6 6 9 19/30 Read block 411 len 40960 20/30 Flush 21/30 Write block 324 len 4096 data: 9 22/30 Read block 393 len 16384 23/30 Read block 548 len 12288 24/30 Write block 300 len 16384 data: 9 8 7 8 25/30 Read block 392 len 36864 26/30 Flush 27/30 Write block 355 len 24576 data: 6 6 5 4 5 4 28/30 Write block 407 len 16384 data: 9 11 10 11 29/30 Read block 404 len 28672 30/30 Read block 196 len 12288 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:34.953017629Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 140 140 140 F F F 1 020-039 89 89 89 139 139 139 F F F 2 040-059 90 90 90 141 141 141 F F F 3 060-079 90 90 90 141 141 141 F F F 4 080-099 89 89 89 139 139 139 F F F 5 100-119 89 89 89 139 139 139 F F F 6 120-139 84 84 84 132 132 132 F F F 7 140-159 90 90 90 140 140 140 F F F 8 160-179 90 90 90 140 140 140 F F F 9 180-199 90 90 90 141 141 141 F F F 10 200-219 90 90 90 141 141 141 F F F 11 220-239 89 89 89 139 139 139 F F F 12 240-259 89 89 89 139 139 139 F F F 13 260-279 84 84 84 132 132 132 F F F 14 280-299 89 89 89 139 139 139 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 90 90 90 142 142 142 F F F 17 340-359 84 84 84 132 132 132 T T T 18 360-379 89 89 89 139 139 139 T T T 19 380-399 90 90 90 140 140 140 F F F 20 400-419 90 90 90 141 141 141 T T T 21 420-439 89 89 89 139 139 139 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 89 89 89 139 139 139 F F F 24 480-499 89 89 89 139 139 139 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 89 89 89 139 139 139 F F F 27 540-559 90 90 90 141 141 141 F F F 28 560-579 90 90 90 140 140 140 F F F 29 580-599 89 89 89 139 139 139 F F F Max gen: 90, Max flush: 142 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 140 140 140 F F F 1 020-039 89 89 89 139 139 139 F F F 2 040-059 90 90 90 141 141 141 F F F 3 060-079 90 90 90 141 141 141 F F F 4 080-099 89 89 89 139 139 139 F F F 5 100-119 89 89 89 139 139 139 F F F 6 120-139 84 84 84 132 132 132 F F F 7 140-159 90 90 90 140 140 140 F F F 8 160-179 90 90 90 140 140 140 F F F 9 180-199 90 90 90 141 141 141 F F F 10 200-219 90 90 90 141 141 141 F F F 11 220-239 89 89 89 139 139 139 F F F 12 240-259 89 89 89 139 139 139 F F F 13 260-279 84 84 84 132 132 132 F F F 14 280-299 89 89 89 139 139 139 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 90 90 90 142 142 142 F F F 17 340-359 91 91 91 143 143 143 F F F 18 360-379 91 91 91 143 143 143 F F F 19 380-399 90 90 90 140 140 140 F F F 20 400-419 91 91 91 143 143 143 F F F 21 420-439 89 89 89 139 139 139 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 89 89 89 139 139 139 F F F 24 480-499 89 89 89 139 139 139 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 89 89 89 139 139 139 F F F 27 540-559 90 90 90 141 141 141 F F F 28 560-579 90 90 90 140 140 140 F F F 29 580-599 89 89 89 139 139 139 F F F Max gen: 91, Max flush: 143 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.177484928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:39.220708507Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.221097626Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.252774548Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.253359577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.253391883Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.253405522Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.253419651Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560} {"msg":"Crucible 60bf10ed-f214-46e4-9586-1c84f557c932 has session id: f7dc7a7b-8d28-415f-a07e-98e9e35617a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.253947755Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Upstairs opts: Upstairs UUID: 60bf10ed-f214-46e4-9586-1c84f557c932, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.253983301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Crucible stats registered with UUID: 60bf10ed-f214-46e4-9586-1c84f557c932","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.25399943Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:39.254016378Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254456262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.25447858Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560} {"msg":"60bf10ed-f214-46e4-9586-1c84f557c932 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254544502Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254561391Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"0","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254588848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"0","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254602806Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"1","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254616665Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"1","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254629473Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"2","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254643112Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"2","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254655691Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254668989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254682318Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:33270)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254754491Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:40485)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254783517Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:51002)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.254901875Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.255490743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"1","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.255528999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"2","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.255687093Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"0","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256168382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"1","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256195769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"2","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256210738Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"0","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256740252Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256866789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[0]R flush_numbers[0..12]: [140, 139, 141, 141, 139, 139, 132, 140, 140, 141, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256897425Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[0]R generation[0..12]: [90, 89, 90, 90, 89, 89, 84, 90, 90, 90, 90, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256912714Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256927842Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[1]R flush_numbers[0..12]: [140, 139, 141, 141, 139, 139, 132, 140, 140, 141, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256942091Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[1]R generation[0..12]: [90, 89, 90, 90, 89, 89, 84, 90, 90, 90, 90, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256955489Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256969138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[2]R flush_numbers[0..12]: [140, 139, 141, 141, 139, 139, 132, 140, 140, 141, 141, 139]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.256986316Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[2]R generation[0..12]: [90, 89, 90, 90, 89, 89, 84, 90, 90, 90, 90, 89]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257000164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257013663Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257026392Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.25703906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257051619Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257064038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"":"downstairs","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257076696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257088335Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"0","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257101314Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"1","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257122302Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"client":"2","":"downstairs","session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.2571351Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257147119Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"60bf10ed-f214-46e4-9586-1c84f557c932 is now active with session: f7dc7a7b-8d28-415f-a07e-98e9e35617a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257159588Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257171277Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560,"session_id":"f7dc7a7b-8d28-415f-a07e-98e9e35617a6"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257184975Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560} {"msg":"Activated sub_volume 60bf10ed-f214-46e4-9586-1c84f557c932","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:39.257196084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 430 len 28672 data: 15 14 13 12 11 12 9 02/30 Write block 281 len 8192 data: 9 11 03/30 Read block 445 len 20480 04/30 Flush 05/30 Write block 153 len 20480 data: 8 7 6 6 7 06/30 Write block 173 len 12288 data: 7 11 10 07/30 Write block 420 len 24576 data: 7 5 6 4 4 6 08/30 Read block 133 len 28672 09/30 Read block 509 len 16384 10/30 Write block 387 len 36864 data: 7 7 7 5 7 7 8 8 9 11/30 Write block 121 len 16384 data: 4 4 4 4 12/30 Write block 235 len 16384 data: 6 6 6 6 13/30 Read block 41 len 12288 14/30 Read block 402 len 24576 15/30 Read block 466 len 12288 16/30 Read block 337 len 24576 17/30 Write block 470 len 12288 data: 15 15 13 18/30 Write block 147 len 40960 data: 9 10 8 8 7 8 9 8 7 7 19/30 Read block 470 len 28672 20/30 Read block 418 len 16384 21/30 Read block 358 len 36864 22/30 Write block 520 len 24576 data: 7 7 7 5 5 6 23/30 Read block 363 len 32768 24/30 Read block 279 len 16384 25/30 Read block 369 len 4096 26/30 Write block 185 len 4096 data: 9 27/30 Read block 226 len 40960 28/30 Read block 565 len 40960 29/30 Read block 243 len 32768 30/30 Write block 563 len 8192 data: 7 7 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 5 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 9 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:39.276586117Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 140 140 140 F F F 1 020-039 89 89 89 139 139 139 F F F 2 040-059 90 90 90 141 141 141 F F F 3 060-079 90 90 90 141 141 141 F F F 4 080-099 89 89 89 139 139 139 F F F 5 100-119 89 89 89 139 139 139 F F F 6 120-139 84 84 84 132 132 132 T T T 7 140-159 90 90 90 140 140 140 T T T 8 160-179 90 90 90 140 140 140 T T T 9 180-199 90 90 90 141 141 141 T T T 10 200-219 90 90 90 141 141 141 F F F 11 220-239 89 89 89 139 139 139 T T T 12 240-259 89 89 89 139 139 139 F F F 13 260-279 84 84 84 132 132 132 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 90 90 90 142 142 142 F F F 17 340-359 91 91 91 143 143 143 F F F 18 360-379 91 91 91 143 143 143 F F F 19 380-399 90 90 90 140 140 140 T T T 20 400-419 91 91 91 143 143 143 F F F 21 420-439 92 92 92 144 144 144 T T T 22 440-459 89 89 89 139 139 139 F F F 23 460-479 89 89 89 139 139 139 T T T 24 480-499 89 89 89 139 139 139 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 89 89 89 139 139 139 T T T 27 540-559 90 90 90 141 141 141 F F F 28 560-579 90 90 90 140 140 140 T T T 29 580-599 89 89 89 139 139 139 F F F Max gen: 92, Max flush: 144 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 140 140 140 F F F 1 020-039 89 89 89 139 139 139 F F F 2 040-059 90 90 90 141 141 141 F F F 3 060-079 90 90 90 141 141 141 F F F 4 080-099 89 89 89 139 139 139 F F F 5 100-119 89 89 89 139 139 139 F F F 6 120-139 93 93 93 145 145 145 F F F 7 140-159 93 93 93 145 145 145 F F F 8 160-179 93 93 93 145 145 145 F F F 9 180-199 93 93 93 145 145 145 F F F 10 200-219 90 90 90 141 141 141 F F F 11 220-239 93 93 93 145 145 145 F F F 12 240-259 89 89 89 139 139 139 F F F 13 260-279 84 84 84 132 132 132 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 90 90 90 142 142 142 F F F 16 320-339 90 90 90 142 142 142 F F F 17 340-359 91 91 91 143 143 143 F F F 18 360-379 91 91 91 143 143 143 F F F 19 380-399 93 93 93 145 145 145 F F F 20 400-419 91 91 91 143 143 143 F F F 21 420-439 93 93 93 145 145 145 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 93 93 93 145 145 145 F F F 24 480-499 89 89 89 139 139 139 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 93 93 93 145 145 145 F F F 27 540-559 90 90 90 141 141 141 F F F 28 560-579 93 93 93 145 145 145 F F F 29 580-599 89 89 89 139 139 139 F F F Max gen: 93, Max flush: 145 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.56754592Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.611990771Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.61267098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.612713205Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.612730374Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.612748212Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568} {"msg":"Crucible db6ba0b2-dd4b-40e5-84f7-1b12b888428a has session id: e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.613262248Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Upstairs opts: Upstairs UUID: db6ba0b2-dd4b-40e5-84f7-1b12b888428a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.613302804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Crucible stats registered with UUID: db6ba0b2-dd4b-40e5-84f7-1b12b888428a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.613328651Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:43.613349759Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.613845996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.613870644Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568} {"msg":"db6ba0b2-dd4b-40e5-84f7-1b12b888428a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.613927868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.613953685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"0","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.613982262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"0","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.61400017Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"1","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614014789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"1","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614029407Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"2","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.61409494Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"2","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614122997Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614147405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614172422Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37579)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614198409Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40879)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614272722Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:51755)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614313697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.614931542Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"0","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.61495775Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"2","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.615647327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"2","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.615679614Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"0","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.616109419Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"1","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.616326916Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.616362202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.616740372Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"1","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[0]R flush_numbers[0..12]: [140, 139, 141, 141, 139, 139, 145, 145, 145, 145, 141, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617281775Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[0]R generation[0..12]: [90, 89, 90, 90, 89, 89, 93, 93, 93, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617302113Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617316972Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[1]R flush_numbers[0..12]: [140, 139, 141, 141, 139, 139, 145, 145, 145, 145, 141, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.61733106Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[1]R generation[0..12]: [90, 89, 90, 90, 89, 89, 93, 93, 93, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617344849Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617358997Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[2]R flush_numbers[0..12]: [140, 139, 141, 141, 139, 139, 145, 145, 145, 145, 141, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617372846Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[2]R generation[0..12]: [90, 89, 90, 90, 89, 89, 93, 93, 93, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617386204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617400513Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617413142Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.61742623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-03-18T15:21:43.617450278Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617463596Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"":"downstairs","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617476515Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617488184Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"0","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617501003Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"1","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617513701Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"client":"2","":"downstairs","session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.61752633Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617538508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"db6ba0b2-dd4b-40e5-84f7-1b12b888428a is now active with session: e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617550967Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617562776Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568,"session_id":"e3fdebe7-3a7f-45b6-b9c8-0192577e3fb3"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617574585Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568} {"msg":"Activated sub_volume db6ba0b2-dd4b-40e5-84f7-1b12b888428a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:43.617585383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 348 len 8192 data: 6 6 02/30 Read block 525 len 28672 03/30 Read block 532 len 40960 04/30 Read block 342 len 16384 05/30 Flush 06/30 Write block 334 len 32768 data: 6 8 8 8 9 8 8 9 07/30 Write block 476 len 4096 data: 6 08/30 Read block 233 len 36864 09/30 Write block 267 len 28672 data: 7 11 10 10 10 11 8 10/30 Write block 102 len 32768 data: 13 12 10 6 5 5 6 6 11/30 Read block 436 len 32768 12/30 Read block 307 len 4096 13/30 Write block 392 len 20480 data: 8 9 9 10 11 14/30 Flush 15/30 Write block 567 len 28672 data: 5 5 5 4 5 7 8 16/30 Flush 17/30 Write block 378 len 12288 data: 10 10 9 18/30 Read block 108 len 20480 19/30 Read block 402 len 28672 20/30 Read block 443 len 12288 21/30 Write block 312 len 40960 data: 8 9 12 11 7 7 7 8 6 8 22/30 Write block 77 len 8192 data: 9 7 23/30 Read block 3 len 32768 24/30 Read block 164 len 4096 25/30 Write block 311 len 12288 data: 8 9 10 26/30 Write block 339 len 20480 data: 9 9 10 7 5 27/30 Write block 580 len 40960 data: 5 9 9 9 12 10 11 12 10 9 28/30 Read block 502 len 40960 29/30 Write block 383 len 20480 data: 7 8 8 6 8 30/30 Read block 549 len 20480 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 3 Done Done Done false 1017 Acked Read 5 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 14 12 12 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) Some(JobId(1015)) Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:43.63769887Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 140 140 140 F F F 1 020-039 89 89 89 139 139 139 F F F 2 040-059 90 90 90 141 141 141 F F F 3 060-079 90 90 90 141 141 141 T T T 4 080-099 89 89 89 139 139 139 F F F 5 100-119 94 94 94 147 147 147 F F F 6 120-139 93 93 93 145 145 145 F F F 7 140-159 93 93 93 145 145 145 F F F 8 160-179 93 93 93 145 145 145 F F F 9 180-199 93 93 93 145 145 145 F F F 10 200-219 90 90 90 141 141 141 F F F 11 220-239 93 93 93 145 145 145 F F F 12 240-259 89 89 89 139 139 139 F F F 13 260-279 94 94 94 147 147 147 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 90 90 90 142 142 142 T T T 16 320-339 94 94 94 147 147 147 T T T 17 340-359 94 94 94 147 147 147 T T T 18 360-379 91 91 91 143 143 143 T T T 19 380-399 94 94 94 147 147 147 T T T 20 400-419 91 91 91 143 143 143 F F F 21 420-439 93 93 93 145 145 145 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 94 94 94 147 147 147 F F F 24 480-499 89 89 89 139 139 139 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 93 93 93 145 145 145 F F F 27 540-559 90 90 90 141 141 141 F F F 28 560-579 94 94 94 148 148 148 F F F 29 580-599 89 89 89 139 139 139 T T T Max gen: 94, Max flush: 148 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 140 140 140 F F F 1 020-039 89 89 89 139 139 139 F F F 2 040-059 90 90 90 141 141 141 F F F 3 060-079 95 95 95 149 149 149 F F F 4 080-099 89 89 89 139 139 139 F F F 5 100-119 94 94 94 147 147 147 F F F 6 120-139 93 93 93 145 145 145 F F F 7 140-159 93 93 93 145 145 145 F F F 8 160-179 93 93 93 145 145 145 F F F 9 180-199 93 93 93 145 145 145 F F F 10 200-219 90 90 90 141 141 141 F F F 11 220-239 93 93 93 145 145 145 F F F 12 240-259 89 89 89 139 139 139 F F F 13 260-279 94 94 94 147 147 147 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 95 95 95 149 149 149 F F F 17 340-359 95 95 95 149 149 149 F F F 18 360-379 95 95 95 149 149 149 F F F 19 380-399 95 95 95 149 149 149 F F F 20 400-419 91 91 91 143 143 143 F F F 21 420-439 93 93 93 145 145 145 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 94 94 94 147 147 147 F F F 24 480-499 89 89 89 139 139 139 F F F 25 500-519 86 86 86 133 133 133 F F F 26 520-539 93 93 93 145 145 145 F F F 27 540-559 90 90 90 141 141 141 F F F 28 560-579 94 94 94 148 148 148 F F F 29 580-599 95 95 95 149 149 149 F F F Max gen: 95, Max flush: 149 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.884974004Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:47.928494421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.928865982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.96087281Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.961496634Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.961531371Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.961545699Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.961560088Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576} {"msg":"Crucible 5de5b069-118d-4969-a66c-6da9be562f3a has session id: 91888b9a-8f7c-4723-b1d3-f179611ee6df","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962134807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Upstairs opts: Upstairs UUID: 5de5b069-118d-4969-a66c-6da9be562f3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962163424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Crucible stats registered with UUID: 5de5b069-118d-4969-a66c-6da9be562f3a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962178143Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:47.962190281Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962622586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962646344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576} {"msg":"5de5b069-118d-4969-a66c-6da9be562f3a active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962711347Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962729665Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"0","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962757792Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"0","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962771591Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"1","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962789239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"1","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962813376Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"2","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962838124Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"2","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962862571Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962887658Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962918175Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"ds_connection connected from Ok(127.0.0.1:57213)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.962949002Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"ds_connection connected from Ok(127.0.0.1:43046)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.96325634Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"ds_connection connected from Ok(127.0.0.1:58394)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.963294996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.963691154Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"2","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.963847527Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"1","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.963972354Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"0","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.964372572Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"1","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.964400069Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"2","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.964674281Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"0","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.96505628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965093447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[0]R flush_numbers[0..12]: [140, 139, 141, 149, 139, 147, 145, 145, 145, 145, 141, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965213864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[0]R generation[0..12]: [90, 89, 90, 95, 89, 94, 93, 93, 93, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965236572Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965255219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[1]R flush_numbers[0..12]: [140, 139, 141, 149, 139, 147, 145, 145, 145, 145, 141, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965269428Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[1]R generation[0..12]: [90, 89, 90, 95, 89, 94, 93, 93, 93, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965282547Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965296055Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[2]R flush_numbers[0..12]: [140, 139, 141, 149, 139, 147, 145, 145, 145, 145, 141, 145]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965309324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[2]R generation[0..12]: [90, 89, 90, 95, 89, 94, 93, 93, 93, 93, 90, 93]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965322232Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965335691Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.9653481Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965361088Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965373557Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":Wait for a query_work_queue command to finish before sending IO 1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965393725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"":"downstairs","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965406404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965418102Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"0","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965431131Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"1","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.9654437Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"client":"2","":"downstairs","session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965458948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965471027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"5de5b069-118d-4969-a66c-6da9be562f3a is now active with session: 91888b9a-8f7c-4723-b1d3-f179611ee6df","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965483436Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965494884Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576,"session_id":"91888b9a-8f7c-4723-b1d3-f179611ee6df"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965506423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576} {"msg":"Activated sub_volume 5de5b069-118d-4969-a66c-6da9be562f3a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:47.965517212Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 5 len 16384 data: 5 7 7 6 02/30 Read block 103 len 40960 03/30 Read block 269 len 32768 04/30 Read block 455 len 28672 05/30 Read block 522 len 16384 06/30 Write block 233 len 28672 data: 7 8 7 7 7 7 9 07/30 Read block 336 len 40960 08/30 Write block 511 len 4096 data: 12 09/30 Read block 188 len 8192 10/30 Write block 555 len 40960 data: 11 10 9 9 6 10 10 9 8 8 11/30 Read block 514 len 20480 12/30 Read block 568 len 20480 13/30 Write block 486 len 40960 data: 10 9 9 7 6 4 6 7 7 5 14/30 Write block 403 len 24576 data: 12 13 13 12 10 12 15/30 Write block 390 len 24576 data: 6 8 9 10 10 11 16/30 Read block 549 len 24576 17/30 Write block 80 len 40960 data: 5 5 6 5 4 4 4 5 6 5 18/30 Write block 343 len 4096 data: 6 19/30 Read block 63 len 36864 20/30 Write block 209 len 28672 data: 7 7 8 9 10 10 10 21/30 Flush 22/30 Write block 479 len 40960 data: 5 6 9 9 8 9 10 11 10 10 23/30 Write block 152 len 8192 data: 9 10 24/30 Flush 25/30 Write block 219 len 40960 data: 14 14 13 11 12 10 9 6 6 5 26/30 Write block 519 len 20480 data: 8 8 8 8 6 27/30 Read block 340 len 24576 28/30 Write block 36 len 32768 data: 9 10 8 6 7 10 9 9 29/30 Write block 411 len 8192 data: 10 8 30/30 Write block 64 len 24576 data: 9 9 10 9 8 9 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:47.986731274Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 89 89 89 139 139 139 T T T 2 040-059 90 90 90 141 141 141 T T T 3 060-079 95 95 95 149 149 149 T T T 4 080-099 96 96 96 150 150 150 F F F 5 100-119 94 94 94 147 147 147 F F F 6 120-139 93 93 93 145 145 145 F F F 7 140-159 96 96 96 151 151 151 F F F 8 160-179 93 93 93 145 145 145 F F F 9 180-199 93 93 93 145 145 145 F F F 10 200-219 96 96 96 150 150 150 T T T 11 220-239 96 96 96 150 150 150 T T T 12 240-259 89 89 89 139 139 139 F F F 13 260-279 94 94 94 147 147 147 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 95 95 95 149 149 149 F F F 17 340-359 96 96 96 150 150 150 F F F 18 360-379 95 95 95 149 149 149 F F F 19 380-399 96 96 96 150 150 150 F F F 20 400-419 96 96 96 150 150 150 T T T 21 420-439 93 93 93 145 145 145 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 96 96 96 151 151 151 F F F 25 500-519 96 96 96 150 150 150 T T T 26 520-539 93 93 93 145 145 145 T T T 27 540-559 96 96 96 150 150 150 F F F 28 560-579 96 96 96 150 150 150 F F F 29 580-599 95 95 95 149 149 149 F F F Max gen: 96, Max flush: 151 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 97 97 97 152 152 152 F F F 2 040-059 97 97 97 152 152 152 F F F 3 060-079 97 97 97 152 152 152 F F F 4 080-099 96 96 96 150 150 150 F F F 5 100-119 94 94 94 147 147 147 F F F 6 120-139 93 93 93 145 145 145 F F F 7 140-159 96 96 96 151 151 151 F F F 8 160-179 93 93 93 145 145 145 F F F 9 180-199 93 93 93 145 145 145 F F F 10 200-219 97 97 97 152 152 152 F F F 11 220-239 97 97 97 152 152 152 F F F 12 240-259 89 89 89 139 139 139 F F F 13 260-279 94 94 94 147 147 147 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 95 95 95 149 149 149 F F F 17 340-359 96 96 96 150 150 150 F F F 18 360-379 95 95 95 149 149 149 F F F 19 380-399 96 96 96 150 150 150 F F F 20 400-419 97 97 97 152 152 152 F F F 21 420-439 93 93 93 145 145 145 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 96 96 96 151 151 151 F F F 25 500-519 97 97 97 152 152 152 F F F 26 520-539 97 97 97 152 152 152 F F F 27 540-559 96 96 96 150 150 150 F F F 28 560-579 96 96 96 150 150 150 F F F 29 580-599 95 95 95 149 149 149 F F F Max gen: 97, Max flush: 152 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.269301789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.313066381Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.313717283Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.313753279Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.313767798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.313782146Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584} {"msg":"Crucible 40060261-53c7-4c7e-a294-95b49cfecddd has session id: 241bd2f6-5f52-48db-bf30-a72a61bcb347","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.314355316Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Upstairs opts: Upstairs UUID: 40060261-53c7-4c7e-a294-95b49cfecddd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.314389893Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Crucible stats registered with UUID: 40060261-53c7-4c7e-a294-95b49cfecddd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.31441605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:52.314438267Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.314830386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.314853514Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584} {"msg":"40060261-53c7-4c7e-a294-95b49cfecddd active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.314915217Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.314932555Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"0","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.314959253Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"0","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.31498653Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"1","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315054043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"1","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315083329Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315112186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315140854Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.31516914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315196568Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:52.315224785Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:52.315252122Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"ds_connection connected from Ok(127.0.0.1:38307)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315277879Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"ds_connection connected from Ok(127.0.0.1:42147)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315377949Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:21:52.315412055Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315434543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315811643Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"0","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.315854179Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"1","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.316403961Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"1","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.316513449Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"0","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.316991399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:21:52.317092638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.316857354Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"ds_connection connected from Ok(127.0.0.1:44083)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.317079831Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.318075316Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.318670354Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[0]R flush_numbers[0..12]: [150, 152, 152, 152, 150, 147, 145, 151, 145, 145, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319259582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[0]R generation[0..12]: [96, 97, 97, 97, 96, 94, 93, 96, 93, 93, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319289429Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319308637Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[1]R flush_numbers[0..12]: [150, 152, 152, 152, 150, 147, 145, 151, 145, 145, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319325915Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[1]R generation[0..12]: [96, 97, 97, 97, 96, 94, 93, 96, 93, 93, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319342483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319366821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[2]R flush_numbers[0..12]: [150, 152, 152, 152, 150, 147, 145, 151, 145, 145, 152, 152]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319384329Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[2]R generation[0..12]: [96, 97, 97, 97, 96, 94, 93, 96, 93, 93, 97, 97]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319401187Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319418495Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319440963Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319457601Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319473379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319489098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"":"downstairs","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319504936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319519614Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"0","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319537363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"1","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319553381Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"client":"2","":"downstairs","session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319569329Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319584138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"40060261-53c7-4c7e-a294-95b49cfecddd is now active with session: 241bd2f6-5f52-48db-bf30-a72a61bcb347","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319603816Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319619814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584,"session_id":"241bd2f6-5f52-48db-bf30-a72a61bcb347"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319632593Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584} {"msg":"Activated sub_volume 40060261-53c7-4c7e-a294-95b49cfecddd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:02.319643892Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 8192 data: 8 7 02/30 Read block 554 len 16384 03/30 Write block 405 len 36864 data: 14 13 11 13 11 12 11 9 8 04/30 Write block 27 len 36864 data: 3 5 7 9 10 10 10 10 10 05/30 Read block 72 len 40960 06/30 Write block 390 len 12288 data: 7 9 10 07/30 Write block 558 len 28672 data: 10 7 11 11 10 9 9 08/30 Write block 357 len 20480 data: 6 5 6 5 5 09/30 Read block 153 len 32768 10/30 Write block 532 len 12288 data: 6 7 9 11/30 Read block 112 len 40960 12/30 Read block 110 len 8192 13/30 Read block 440 len 4096 14/30 Read block 337 len 20480 15/30 Write block 213 len 36864 data: 11 11 11 12 13 12 15 15 14 16/30 Read block 439 len 12288 17/30 Read block 328 len 24576 18/30 Write block 148 len 16384 data: 11 9 9 8 19/30 Read block 227 len 40960 20/30 Write block 52 len 20480 data: 9 8 10 10 11 21/30 Write block 180 len 24576 data: 7 8 8 10 10 10 22/30 Read block 276 len 12288 23/30 Read block 268 len 20480 24/30 Write block 270 len 40960 data: 11 11 12 9 8 8 5 5 5 7 25/30 Write block 121 len 20480 data: 5 5 5 5 3 26/30 Write block 560 len 16384 data: 12 12 11 10 27/30 Write block 397 len 20480 data: 10 11 12 11 11 28/30 Write block 30 len 12288 data: 10 11 11 29/30 Write block 154 len 32768 data: 9 8 8 8 7 7 8 8 30/30 Write block 345 len 8192 data: 5 3 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Write 5 Sent Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 6 4 4 14 Done 24 26 26 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:02.339184169Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 97 97 97 152 152 152 T T T 2 040-059 97 97 97 152 152 152 T T T 3 060-079 97 97 97 152 152 152 F F F 4 080-099 96 96 96 150 150 150 F F F 5 100-119 94 94 94 147 147 147 F F F 6 120-139 93 93 93 145 145 145 T T T 7 140-159 96 96 96 151 151 151 T T T 8 160-179 93 93 93 145 145 145 T T T 9 180-199 93 93 93 145 145 145 T T T 10 200-219 97 97 97 152 152 152 T T T 11 220-239 97 97 97 152 152 152 T T T 12 240-259 89 89 89 139 139 139 F F F 13 260-279 94 94 94 147 147 147 T T T 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 95 95 95 149 149 149 F F F 17 340-359 96 96 96 150 150 150 T T T 18 360-379 95 95 95 149 149 149 T T T 19 380-399 96 96 96 150 150 150 T T T 20 400-419 97 97 97 152 152 152 T T T 21 420-439 93 93 93 145 145 145 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 96 96 96 151 151 151 F F F 25 500-519 97 97 97 152 152 152 F F F 26 520-539 97 97 97 152 152 152 T T T 27 540-559 96 96 96 150 150 150 T T T 28 560-579 96 96 96 150 150 150 T T T 29 580-599 95 95 95 149 149 149 F F F Max gen: 97, Max flush: 152 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 98 98 98 153 153 153 F F F 2 040-059 98 98 98 153 153 153 F F F 3 060-079 97 97 97 152 152 152 F F F 4 080-099 96 96 96 150 150 150 F F F 5 100-119 94 94 94 147 147 147 F F F 6 120-139 98 98 98 153 153 153 F F F 7 140-159 98 98 98 153 153 153 F F F 8 160-179 98 98 98 153 153 153 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 98 98 98 153 153 153 F F F 11 220-239 98 98 98 153 153 153 F F F 12 240-259 89 89 89 139 139 139 F F F 13 260-279 98 98 98 153 153 153 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 95 95 95 149 149 149 F F F 17 340-359 98 98 98 153 153 153 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 98 98 98 153 153 153 F F F 20 400-419 98 98 98 153 153 153 F F F 21 420-439 93 93 93 145 145 145 F F F 22 440-459 89 89 89 139 139 139 F F F 23 460-479 96 96 96 151 151 151 F F F 24 480-499 96 96 96 151 151 151 F F F 25 500-519 97 97 97 152 152 152 F F F 26 520-539 98 98 98 153 153 153 F F F 27 540-559 98 98 98 153 153 153 F F F 28 560-579 98 98 98 153 153 153 F F F 29 580-599 95 95 95 149 149 149 F F F Max gen: 98, Max flush: 153 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.638787438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:16.68083703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.681205042Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.712094557Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.712660777Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.712693364Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.712706662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.712721811Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592} {"msg":"Crucible 4d58680c-e976-441a-a065-c2c2ae308711 has session id: 6184a04d-48f3-4139-8d5b-83d59ff0d4dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713280712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Upstairs opts: Upstairs UUID: 4d58680c-e976-441a-a065-c2c2ae308711, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713323238Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Crucible stats registered with UUID: 4d58680c-e976-441a-a065-c2c2ae308711","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713338996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:16.713352435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713723116Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713751033Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592} {"msg":"4d58680c-e976-441a-a065-c2c2ae308711 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713808367Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713825375Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"0","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713852682Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"0","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713866871Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"1","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713880649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"1","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713893328Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"2","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713906906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"2","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713920705Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713934124Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.713947382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41539)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.714023114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:65225)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.714207945Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"ds_connection connected from Ok(127.0.0.1:51672)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.714257639Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.71472655Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"1","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.714771605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"2","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.714881894Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"0","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.715385641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"2","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.715424947Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"1","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.715455924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"0","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716027513Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716055261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[0]R flush_numbers[0..12]: [150, 153, 153, 152, 150, 147, 153, 153, 153, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716086187Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[0]R generation[0..12]: [96, 98, 98, 97, 96, 94, 98, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716101546Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716115714Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[1]R flush_numbers[0..12]: [150, 153, 153, 152, 150, 147, 153, 153, 153, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716129703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[1]R generation[0..12]: [96, 98, 98, 97, 96, 94, 98, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716143671Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.71615748Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[2]R flush_numbers[0..12]: [150, 153, 153, 152, 150, 147, 153, 153, 153, 153, 153, 153]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716171029Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[2]R generation[0..12]: [96, 98, 98, 97, 96, 94, 98, 98, 98, 98, 98, 98]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716188347Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716202455Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Max found gen is 99","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716214884Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Generation requested: 100 >= found:99","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716228523Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716240961Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716258399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"":"downstairs","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:22:16.716281987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716300655Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"0","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716313774Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"1","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716326292Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"client":"2","":"downstairs","session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716340861Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716362069Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"4d58680c-e976-441a-a065-c2c2ae308711 is now active with session: 6184a04d-48f3-4139-8d5b-83d59ff0d4dc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716379197Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716392045Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592,"session_id":"6184a04d-48f3-4139-8d5b-83d59ff0d4dc"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716406654Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592} {"msg":"Activated sub_volume 4d58680c-e976-441a-a065-c2c2ae308711","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:16.716425782Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 589 len 36864 data: 10 7 7 6 5 5 6 6 4 03/30 Write block 528 len 16384 data: 7 6 6 5 04/30 Read block 36 len 12288 05/30 Write block 459 len 36864 data: 9 10 12 13 12 11 9 12 13 06/30 Write block 123 len 32768 data: 6 6 4 4 4 7 6 7 07/30 Write block 234 len 12288 data: 9 8 8 08/30 Read block 44 len 12288 09/30 Write block 422 len 28672 data: 7 5 5 7 7 8 9 10/30 Read block 0 len 12288 11/30 Read block 262 len 24576 12/30 Read block 552 len 32768 13/30 Write block 247 len 8192 data: 8 8 14/30 Read block 183 len 32768 15/30 Write block 510 len 16384 data: 11 13 11 10 16/30 Write block 557 len 28672 data: 10 11 8 13 13 12 11 17/30 Write block 477 len 16384 data: 5 5 6 7 18/30 Write block 205 len 28672 data: 6 7 6 8 8 8 9 19/30 Read block 250 len 12288 20/30 Read block 552 len 12288 21/30 Write block 117 len 8192 data: 5 5 22/30 Write block 541 len 40960 data: 6 8 10 9 8 9 8 9 10 10 23/30 Write block 166 len 28672 data: 10 11 10 7 8 7 8 24/30 Write block 392 len 24576 data: 11 11 11 12 12 11 25/30 Write block 588 len 36864 data: 11 11 8 8 7 6 6 7 7 26/30 Read block 424 len 24576 27/30 Read block 190 len 40960 28/30 Write block 84 len 24576 data: 5 5 5 6 7 6 29/30 Read block 297 len 28672 30/30 Read block 22 len 20480 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 9 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 8 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 29 28 28 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:16.736171177Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 98 98 98 153 153 153 F F F 2 040-059 98 98 98 153 153 153 F F F 3 060-079 97 97 97 152 152 152 F F F 4 080-099 96 96 96 150 150 150 T T T 5 100-119 94 94 94 147 147 147 T T T 6 120-139 98 98 98 153 153 153 T T T 7 140-159 98 98 98 153 153 153 F F F 8 160-179 98 98 98 153 153 153 T T T 9 180-199 98 98 98 153 153 153 F F F 10 200-219 98 98 98 153 153 153 T T T 11 220-239 98 98 98 153 153 153 T T T 12 240-259 89 89 89 139 139 139 T T T 13 260-279 98 98 98 153 153 153 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 95 95 95 149 149 149 F F F 17 340-359 98 98 98 153 153 153 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 98 98 98 153 153 153 T T T 20 400-419 98 98 98 153 153 153 F F F 21 420-439 93 93 93 145 145 145 T T T 22 440-459 89 89 89 139 139 139 T T T 23 460-479 96 96 96 151 151 151 T T T 24 480-499 96 96 96 151 151 151 T T T 25 500-519 97 97 97 152 152 152 T T T 26 520-539 98 98 98 153 153 153 T T T 27 540-559 98 98 98 153 153 153 T T T 28 560-579 98 98 98 153 153 153 T T T 29 580-599 95 95 95 149 149 149 T T T Max gen: 98, Max flush: 153 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 98 98 98 153 153 153 F F F 2 040-059 98 98 98 153 153 153 F F F 3 060-079 97 97 97 152 152 152 F F F 4 080-099 99 99 99 154 154 154 F F F 5 100-119 99 99 99 154 154 154 F F F 6 120-139 99 99 99 154 154 154 F F F 7 140-159 98 98 98 153 153 153 F F F 8 160-179 99 99 99 154 154 154 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 99 99 99 154 154 154 F F F 11 220-239 99 99 99 154 154 154 F F F 12 240-259 99 99 99 154 154 154 F F F 13 260-279 98 98 98 153 153 153 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 95 95 95 149 149 149 F F F 17 340-359 98 98 98 153 153 153 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 99 99 99 154 154 154 F F F 20 400-419 98 98 98 153 153 153 F F F 21 420-439 99 99 99 154 154 154 F F F 22 440-459 99 99 99 154 154 154 F F F 23 460-479 99 99 99 154 154 154 F F F 24 480-499 99 99 99 154 154 154 F F F 25 500-519 99 99 99 154 154 154 F F F 26 520-539 99 99 99 154 154 154 F F F 27 540-559 99 99 99 154 154 154 F F F 28 560-579 99 99 99 154 154 154 F F F 29 580-599 99 99 99 154 154 154 F F F Max gen: 99, Max flush: 154 Begin loop 51 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.061036889Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.105132447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.10576399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.105799747Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.105813745Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.105828814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600} {"msg":"Crucible bbf98c54-b961-4a9e-8762-ff7f3f7df025 has session id: 70bf3395-3fe8-4853-b48d-3e70a791892b","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.106314863Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Upstairs opts: Upstairs UUID: bbf98c54-b961-4a9e-8762-ff7f3f7df025, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.106353288Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Crucible stats registered with UUID: bbf98c54-b961-4a9e-8762-ff7f3f7df025","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.106375276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:21.106394474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.106836928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.106868404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600} {"msg":"bbf98c54-b961-4a9e-8762-ff7f3f7df025 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.106924028Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.106955795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"0","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.10700987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"0","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107042046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"1","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107084911Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"1","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107122348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107158004Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107209738Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107248844Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107379931Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:21.107421096Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:21.107444344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:21.107463962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49591)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.10748041Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45727)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107497358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107517586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107868099Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"1","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.107893637Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"0","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.10853313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"0","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.108567326Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"1","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.109121118Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:21.109183371Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.108942949Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58161)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.109160246Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.110579827Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111133129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[0]R flush_numbers[0..12]: [150, 153, 153, 152, 154, 154, 154, 153, 154, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111718107Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[0]R generation[0..12]: [96, 98, 98, 97, 99, 99, 99, 98, 99, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111746174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111765172Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[1]R flush_numbers[0..12]: [150, 153, 153, 152, 154, 154, 154, 153, 154, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111782821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[1]R generation[0..12]: [96, 98, 98, 97, 99, 99, 99, 98, 99, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111799839Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111817047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[2]R flush_numbers[0..12]: [150, 153, 153, 152, 154, 154, 154, 153, 154, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111839184Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[2]R generation[0..12]: [96, 98, 98, 97, 99, 99, 99, 98, 99, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111856263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111873101Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111888749Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Generation requested: 102 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111904508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111920126Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111935754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"":"downstairs","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111951973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111966421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"0","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.111982649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"client":"1","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.112002817Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"clientWait for a query_work_queue command to finish before sending IO ":"2","":"downstairs","session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.112036484Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.112052772Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"bbf98c54-b961-4a9e-8762-ff7f3f7df025 is now active with session: 70bf3395-3fe8-4853-b48d-3e70a791892b","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.11206889Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.112083689Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600,"session_id":"70bf3395-3fe8-4853-b48d-3e70a791892b"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.112101567Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600} {"msg":"Activated sub_volume bbf98c54-b961-4a9e-8762-ff7f3f7df025","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:31.112115266Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 593 len 4096 data: 7 02/30 Flush 03/30 Write block 564 len 24576 data: 10 7 8 6 6 6 04/30 Write block 501 len 4096 data: 6 05/30 Read block 328 len 32768 06/30 Read block 321 len 32768 07/30 Read block 321 len 24576 08/30 Read block 572 len 24576 09/30 Write block 398 len 20480 data: 12 13 12 12 12 10/30 Write block 344 len 28672 data: 5 6 4 5 7 7 6 11/30 Write block 112 len 4096 data: 7 12/30 Read block 499 len 16384 13/30 Read block 326 len 24576 14/30 Write block 569 len 4096 data: 7 15/30 Read block 310 len 12288 16/30 Read block 69 len 40960 17/30 Read block 427 len 40960 18/30 Write block 414 len 28672 data: 9 8 10 11 8 10 8 19/30 Read block 561 len 40960 20/30 Write block 557 len 24576 data: 11 12 9 14 14 13 21/30 Read block 146 len 4096 22/30 Read block 254 len 28672 23/30 Write block 112 len 12288 data: 8 6 6 24/30 Read block 24 len 24576 25/30 Read block 18 len 16384 26/30 Write block 338 len 16384 data: 10 10 10 11 27/30 Write block 121 len 24576 data: 6 6 7 7 5 5 28/30 Write block 521 len 28672 data: 9 9 7 6 7 8 7 29/30 Read block 83 len 4096 30/30 Write block 351 len 4096 data: 6 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 6 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:31.131647434Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 98 98 98 153 153 153 F F F 2 040-059 98 98 98 153 153 153 F F F 3 060-079 97 97 97 152 152 152 F F F 4 080-099 99 99 99 154 154 154 F F F 5 100-119 99 99 99 154 154 154 T T T 6 120-139 99 99 99 154 154 154 T T T 7 140-159 98 98 98 153 153 153 F F F 8 160-179 99 99 99 154 154 154 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 99 99 99 154 154 154 F F F 11 220-239 99 99 99 154 154 154 F F F 12 240-259 99 99 99 154 154 154 F F F 13 260-279 98 98 98 153 153 153 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 95 95 95 149 149 149 T T T 17 340-359 98 98 98 153 153 153 T T T 18 360-379 98 98 98 153 153 153 F F F 19 380-399 99 99 99 154 154 154 T T T 20 400-419 98 98 98 153 153 153 T T T 21 420-439 99 99 99 154 154 154 T T T 22 440-459 99 99 99 154 154 154 F F F 23 460-479 99 99 99 154 154 154 F F F 24 480-499 99 99 99 154 154 154 F F F 25 500-519 99 99 99 154 154 154 T T T 26 520-539 99 99 99 154 154 154 T T T 27 540-559 99 99 99 154 154 154 T T T 28 560-579 99 99 99 154 154 154 T T T 29 580-599 102 102 102 155 155 155 F F F Max gen: 102, Max flush: 155 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 98 98 98 153 153 153 F F F 2 040-059 98 98 98 153 153 153 F F F 3 060-079 97 97 97 152 152 152 F F F 4 080-099 99 99 99 154 154 154 F F F 5 100-119 103 103 103 156 156 156 F F F 6 120-139 103 103 103 156 156 156 F F F 7 140-159 98 98 98 153 153 153 F F F 8 160-179 99 99 99 154 154 154 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 99 99 99 154 154 154 F F F 11 220-239 99 99 99 154 154 154 F F F 12 240-259 99 99 99 154 154 154 F F F 13 260-279 98 98 98 153 153 153 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 95 95 95 149 149 149 F F F 16 320-339 103 103 103 156 156 156 F F F 17 340-359 103 103 103 156 156 156 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 103 103 103 156 156 156 F F F 20 400-419 103 103 103 156 156 156 F F F 21 420-439 103 103 103 156 156 156 F F F 22 440-459 99 99 99 154 154 154 F F F 23 460-479 99 99 99 154 154 154 F F F 24 480-499 99 99 99 154 154 154 F F F 25 500-519 103 103 103 156 156 156 F F F 26 520-539 103 103 103 156 156 156 F F F 27 540-559 103 103 103 156 156 156 F F F 28 560-579 103 103 103 156 156 156 F F F 29 580-599 102 102 102 155 155 155 F F F Max gen: 103, Max flush: 156 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.397294519Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:45.43992771Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.44030669Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.471070109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.471708062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.471741688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.471755426Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.471769715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608} {"msg":"Crucible d72ee821-e447-48e0-9967-a5a2d74bbba9 has session id: f290d2a7-be2b-4fbf-9485-bd6a5c11842f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472303569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Upstairs opts: Upstairs UUID: d72ee821-e447-48e0-9967-a5a2d74bbba9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472344924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Crucible stats registered with UUID: d72ee821-e447-48e0-9967-a5a2d74bbba9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472369132Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:45.47238684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472743953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.4727658Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608} {"msg":"d72ee821-e447-48e0-9967-a5a2d74bbba9 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472833703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472854941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"0","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472882708Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"0","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472904676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"1","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472930623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"1","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.47295543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"2","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.472983648Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"2","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.473044641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.473074358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.473100965Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61492)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.473127852Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"ds_connection connected from Ok(127.0.0.1:38687)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.47324446Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33039)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.473271327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.47372393Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"1","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.473762715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"2","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.473971644Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"0","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.474462162Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"1","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.474492799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"2","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.474510027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"0","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475133411Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475159239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[0]R flush_numbers[0..12]: [150, 153, 153, 152, 154, 156, 156, 153, 154, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475174247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[0]R generation[0..12]: [96, 98, 98, 97, 99, 103, 103, 98, 99, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475188956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475203264Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[1]R flush_numbers[0..12]: [150, 153, 153, 152, 154, 156, 156, 153, 154, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475216923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[1]R generation[0..12]: [96, 98, 98, 97, 99, 103, 103, 98, 99, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475230181Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.47524384Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[2]R flush_numbers[0..12]: [150, 153, 153, 152, 154, 156, 156, 153, 154, 153, 154, 154]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475257238Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[2]R generation[0..12]: [96, 98, 98, 97, 99, 103, 103, 98, 99, 98, 99, 99]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475270397Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475284196Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475300064Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475320532Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475344389Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475368987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"":"downstairs","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475385835Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475397564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"0","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T15:22:45.475419612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"1","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475450358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"client":"2","":"downstairs","session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475475706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475489674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"d72ee821-e447-48e0-9967-a5a2d74bbba9 is now active with session: f290d2a7-be2b-4fbf-9485-bd6a5c11842f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475502273Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475520001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608,"session_id":"f290d2a7-be2b-4fbf-9485-bd6a5c11842f"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475542848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608} {"msg":"Activated sub_volume d72ee821-e447-48e0-9967-a5a2d74bbba9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:45.475564026Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 558 len 20480 data: 13 10 15 15 14 02/30 Read block 396 len 32768 03/30 Write block 552 len 40960 data: 12 12 10 12 11 12 14 11 16 16 04/30 Write block 53 len 8192 data: 9 11 05/30 Write block 46 len 40960 data: 9 10 8 9 9 8 10 10 12 11 06/30 Write block 518 len 12288 data: 10 9 9 07/30 Flush 08/30 Write block 340 len 20480 data: 11 12 9 8 6 09/30 Write block 315 len 32768 data: 12 8 8 8 9 7 9 6 10/30 Read block 474 len 32768 11/30 Write block 435 len 32768 data: 13 10 8 8 6 7 6 6 12/30 Flush 13/30 Write block 396 len 4096 data: 13 14/30 Read block 40 len 36864 15/30 Read block 329 len 8192 16/30 Read block 60 len 36864 17/30 Read block 459 len 20480 18/30 Read block 514 len 12288 19/30 Write block 403 len 24576 data: 13 14 15 14 12 14 20/30 Read block 520 len 32768 21/30 Read block 210 len 24576 22/30 Flush 23/30 Write block 119 len 28672 data: 6 5 7 7 8 8 6 24/30 Read block 284 len 32768 25/30 Read block 491 len 16384 26/30 Read block 295 len 12288 27/30 Write block 235 len 40960 data: 9 9 8 8 10 9 8 8 10 8 28/30 Write block 454 len 28672 data: 9 9 11 10 8 10 11 29/30 Write block 62 len 16384 data: 10 10 10 10 30/30 Read block 47 len 20480 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 6 8 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:45.497441348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 98 98 98 153 153 153 F F F 2 040-059 104 104 104 157 157 157 F F F 3 060-079 97 97 97 152 152 152 T T T 4 080-099 99 99 99 154 154 154 F F F 5 100-119 103 103 103 156 156 156 T T T 6 120-139 103 103 103 156 156 156 T T T 7 140-159 98 98 98 153 153 153 F F F 8 160-179 99 99 99 154 154 154 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 99 99 99 154 154 154 F F F 11 220-239 99 99 99 154 154 154 T T T 12 240-259 99 99 99 154 154 154 T T T 13 260-279 98 98 98 153 153 153 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 104 104 104 158 158 158 F F F 17 340-359 104 104 104 158 158 158 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 104 104 104 159 159 159 F F F 20 400-419 104 104 104 159 159 159 F F F 21 420-439 104 104 104 158 158 158 F F F 22 440-459 104 104 104 158 158 158 T T T 23 460-479 99 99 99 154 154 154 T T T 24 480-499 99 99 99 154 154 154 F F F 25 500-519 104 104 104 157 157 157 F F F 26 520-539 104 104 104 157 157 157 F F F 27 540-559 104 104 104 157 157 157 F F F 28 560-579 104 104 104 157 157 157 F F F 29 580-599 102 102 102 155 155 155 F F F Max gen: 104, Max flush: 159 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 98 98 98 153 153 153 F F F 2 040-059 104 104 104 157 157 157 F F F 3 060-079 105 105 105 160 160 160 F F F 4 080-099 99 99 99 154 154 154 F F F 5 100-119 105 105 105 160 160 160 F F F 6 120-139 105 105 105 160 160 160 F F F 7 140-159 98 98 98 153 153 153 F F F 8 160-179 99 99 99 154 154 154 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 99 99 99 154 154 154 F F F 11 220-239 105 105 105 160 160 160 F F F 12 240-259 105 105 105 160 160 160 F F F 13 260-279 98 98 98 153 153 153 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 104 104 104 158 158 158 F F F 17 340-359 104 104 104 158 158 158 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 104 104 104 159 159 159 F F F 20 400-419 104 104 104 159 159 159 F F F 21 420-439 104 104 104 158 158 158 F F F 22 440-459 105 105 105 160 160 160 F F F 23 460-479 105 105 105 160 160 160 F F F 24 480-499 99 99 99 154 154 154 F F F 25 500-519 104 104 104 157 157 157 F F F 26 520-539 104 104 104 157 157 157 F F F 27 540-559 104 104 104 157 157 157 F F F 28 560-579 104 104 104 157 157 157 F F F 29 580-599 102 102 102 155 155 155 F F F Max gen: 105, Max flush: 160 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.772390164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.815649419Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.816206871Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.816247906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.816265425Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.816282953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616} {"msg":"Crucible f17d7ce0-bd3f-4d6b-a222-13e123b7296e has session id: 8b1b351b-1d02-4004-a66e-940c1528dffc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.81678346Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Upstairs opts: Upstairs UUID: f17d7ce0-bd3f-4d6b-a222-13e123b7296e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.816834415Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Crucible stats registered with UUID: f17d7ce0-bd3f-4d6b-a222-13e123b7296e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.816863582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:49.816889049Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817323653Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.81735066Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616} {"msg":"f17d7ce0-bd3f-4d6b-a222-13e123b7296e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817433232Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817457039Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"0","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817485006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"0","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817498845Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817512953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817535741Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"2","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817562018Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"2","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817579516Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817593135Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817614853Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:49.817686465Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"ds_connection connected from Ok(127.0.0.1:37846)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817718582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:49.817747329Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54805)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.817774076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:49.817801993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.81782529Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.818358534Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"0","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.818397901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"2","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.819009656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"0","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.819137513Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"2","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.819532771Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:49.819672756Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.819345473Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"ds_connection connected from Ok(127.0.0.1:47773)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.819412066Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.820401272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821027996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[0]R flush_numbers[0..12]: [150, 153, 157, 160, 154, 160, 160, 153, 154, 153, 154, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821627753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[0]R generation[0..12]: [96, 98, 104, 105, 99, 105, 105, 98, 99, 98, 99, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821649051Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821663999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[1]R flush_numbers[0..12]: [150, 153, 157, 160, 154, 160, 160, 153, 154, 153, 154, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821677668Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[1]R generation[0..12]: [96, 98, 104, 105, 99, 105, 105, 98, 99, 98, 99, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821691076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821705295Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[2]R flush_numbers[0..12]: [150, 153, 157, 160, 154, 160, 160, 153, 154, 153, 154, 160]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821718664Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[2]R generation[0..12]: [96, 98, 104, 105, 99, 105, 105, 98, 99, 98, 99, 105]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821731972Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821745521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821758199Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821770928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821783387Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821801675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"":"downstairs","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821815313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821827112Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"0","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821840331Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"1","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821853279Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"client":"2","":"downstairs","session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821866218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821878077Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"f17d7ce0-bd3f-4d6b-a222-13e123b7296e is now active with session: 8b1b351b-1d02-4004-a66e-940c1528dffc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821890765Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} Wait for a query_work_queue command to finish before sending IO {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821909453Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616,"session_id":"8b1b351b-1d02-4004-a66e-940c1528dffc"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821921622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616} {"msg":"Activated sub_volume f17d7ce0-bd3f-4d6b-a222-13e123b7296e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:22:59.821932461Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 28672 data: 9 8 10 10 10 9 9 02/30 Read block 87 len 4096 03/30 Write block 114 len 40960 data: 7 5 6 6 6 7 6 8 8 9 04/30 Write block 277 len 8192 data: 6 6 05/30 Write block 77 len 16384 data: 10 8 5 6 06/30 Read block 528 len 20480 07/30 Read block 392 len 32768 08/30 Read block 452 len 28672 09/30 Write block 25 len 8192 data: 4 4 10/30 Read block 296 len 20480 11/30 Write block 216 len 36864 data: 13 14 13 16 16 15 12 13 11 12/30 Write block 586 len 4096 data: 12 13/30 Read block 120 len 4096 14/30 Write block 402 len 12288 data: 13 14 15 15/30 Write block 454 len 12288 data: 10 10 12 16/30 Read block 50 len 32768 17/30 Write block 455 len 28672 data: 11 13 11 9 11 12 13 18/30 Flush 19/30 Write block 225 len 12288 data: 10 7 7 20/30 Read block 257 len 40960 21/30 Write block 156 len 20480 data: 9 9 8 8 9 22/30 Write block 504 len 32768 data: 7 8 7 7 6 9 12 14 23/30 Write block 346 len 8192 data: 5 6 24/30 Read block 107 len 12288 25/30 Flush 26/30 Write block 576 len 16384 data: 6 6 7 6 27/30 Write block 586 len 36864 data: 13 13 12 12 9 9 8 8 7 28/30 Write block 568 len 8192 data: 7 8 29/30 Write block 417 len 28672 data: 12 9 11 9 6 8 6 30/30 Write block 419 len 20480 data: 12 10 7 9 7 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 9 Sent Done Sent false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 3 4 11 Done 1 2 1 4 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:22:59.841477958Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 106 106 106 161 161 161 F F F 2 040-059 104 104 104 157 157 157 F F F 3 060-079 106 106 106 161 161 161 F F F 4 080-099 106 106 106 161 161 161 F F F 5 100-119 106 106 106 161 161 161 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 106 106 106 162 162 162 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 106 106 106 161 161 161 F F F 11 220-239 106 106 106 162 162 162 F F F 12 240-259 105 105 105 160 160 160 F F F 13 260-279 106 106 106 161 161 161 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 104 104 104 158 158 158 F F F 17 340-359 106 106 106 162 162 162 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 104 104 104 159 159 159 F F F 20 400-419 106 106 106 161 161 161 T T T 21 420-439 104 104 104 158 158 158 T T T 22 440-459 106 106 106 161 161 161 F F F 23 460-479 106 106 106 161 161 161 F F F 24 480-499 99 99 99 154 154 154 F F F 25 500-519 106 106 106 162 162 162 F F F 26 520-539 104 104 104 157 157 157 F F F 27 540-559 104 104 104 157 157 157 F F F 28 560-579 104 104 104 157 157 157 T T T 29 580-599 106 106 106 161 161 161 T T T Max gen: 106, Max flush: 162 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 106 106 106 161 161 161 F F F 2 040-059 104 104 104 157 157 157 F F F 3 060-079 106 106 106 161 161 161 F F F 4 080-099 106 106 106 161 161 161 F F F 5 100-119 106 106 106 161 161 161 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 106 106 106 162 162 162 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 98 98 98 153 153 153 F F F 10 200-219 106 106 106 161 161 161 F F F 11 220-239 106 106 106 162 162 162 F F F 12 240-259 105 105 105 160 160 160 F F F 13 260-279 106 106 106 161 161 161 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 104 104 104 158 158 158 F F F 17 340-359 106 106 106 162 162 162 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 104 104 104 159 159 159 F F F 20 400-419 107 107 107 163 163 163 F F F 21 420-439 107 107 107 163 163 163 F F F 22 440-459 106 106 106 161 161 161 F F F 23 460-479 106 106 106 161 161 161 F F F 24 480-499 99 99 99 154 154 154 F F F 25 500-519 106 106 106 162 162 162 F F F 26 520-539 104 104 104 157 157 157 F F F 27 540-559 104 104 104 157 157 157 F F F 28 560-579 107 107 107 163 163 163 F F F 29 580-599 107 107 107 163 163 163 F F F Max gen: 107, Max flush: 163 Begin loop 54 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.102212717Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.147250576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.147936804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.147967331Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.147981549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.148022995Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624} {"msg":"Crucible 07945b8f-6d89-4e31-8b6f-77058982d74c has session id: a9efe6ca-97b4-4703-9a6a-723c984f4a80","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.148625142Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Upstairs opts: Upstairs UUID: 07945b8f-6d89-4e31-8b6f-77058982d74c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.148664867Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Crucible stats registered with UUID: 07945b8f-6d89-4e31-8b6f-77058982d74c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.148688495Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:04.148708843Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149241477Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149267864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624} {"msg":"07945b8f-6d89-4e31-8b6f-77058982d74c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149356935Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149375193Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"0","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.14940312Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"0","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149417458Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"1","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149439566Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"1","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149464573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"2","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149491731Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"2","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149521577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149568223Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149596549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"ds_connection connected from Ok(127.0.0.1:44756)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149655713Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"ds_connection connected from Ok(127.0.0.1:37434)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149696739Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"ds_connection connected from Ok(127.0.0.1:37408)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.149722866Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.150361249Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"0","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.150399115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"1","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.151089533Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"1","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.151221019Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"2","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.151242407Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"0","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.15178421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.151896538Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"2","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.151928084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[0]R flush_numbers[0..12]: [150, 161, 157, 161, 161, 161, 161, 162, 162, 153, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152484386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[0]R generation[0..12]: [96, 106, 104, 106, 106, 106, 106, 106, 106, 98, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152512923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152529801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[1]R flush_numbers[0..12]: [150, 161, 157, 161, 161, 161, 161, 162, 162, 153, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.15254404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[1]R generation[0..12]: [96, 106, 104, 106, 106, 106, 106, 106, 106, 98, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152557898Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152571757Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[2]R flush_numbers[0..12]: [150, 161, 157, 161, 161, 161, 161, 162, 162, 153, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152590265Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[2]R generation[0..12]: [96, 106, 104, 106, 106, 106, 106, 106, 106, 98, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152605104Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152619262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.15263219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152645269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152665557Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152678486Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"":"downstairs","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152691144Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152702963Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"0","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152716822Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"1","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.15272976Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"client":"2","":"downstairs","session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152742579Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152754588Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"07945b8f-6d89-4e31-8b6f-77058982d74c is now active with session: a9efe6ca-97b4-4703-9a6a-723c984f4a80","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152767156Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152782185Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624,"session_id":"a9efe6ca-97b4-4703-9a6a-723c984f4a80"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152797283Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624} {"msg":"Activated sub_volume 07945b8f-6d89-4e31-8b6f-77058982d74c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:04.152809192Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 190 len 24576 data: 6 10 10 10 9 8 03/30 Write block 450 len 40960 data: 9 10 9 8 11 12 14 12 10 12 04/30 Read block 436 len 16384 05/30 Read block 279 len 12288 06/30 Read block 271 len 20480 07/30 Read block 278 len 24576 08/30 Read block 374 len 24576 09/30 Flush 10/30 Write block 333 len 12288 data: 7 7 9 11/30 Write block 574 len 32768 data: 7 7 7 7 8 7 6 10 12/30 Flush 13/30 Write block 589 len 32768 data: 13 10 10 9 9 8 8 8 14/30 Write block 48 len 20480 data: 9 10 10 9 11 15/30 Write block 485 len 20480 data: 11 12 11 11 8 16/30 Read block 271 len 24576 17/30 Read block 339 len 28672 18/30 Write block 96 len 16384 data: 10 9 9 13 19/30 Read block 179 len 24576 20/30 Write block 330 len 20480 data: 9 8 7 8 8 21/30 Read block 304 len 24576 22/30 Write block 560 len 8192 data: 17 17 23/30 Flush 24/30 Write block 494 len 32768 data: 8 6 5 5 4 4 6 7 25/30 Read block 550 len 20480 26/30 Read block 564 len 20480 27/30 Read block 12 len 32768 28/30 Write block 22 len 20480 data: 6 6 4 5 5 29/30 Read block 141 len 40960 30/30 Read block 462 len 32768 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 8 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 7 6 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:04.174012844Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 106 106 106 161 161 161 T T T 2 040-059 108 108 108 167 167 167 F F F 3 060-079 106 106 106 161 161 161 F F F 4 080-099 108 108 108 167 167 167 F F F 5 100-119 106 106 106 161 161 161 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 106 106 106 162 162 162 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 108 108 108 165 165 165 F F F 10 200-219 106 106 106 161 161 161 F F F 11 220-239 106 106 106 162 162 162 F F F 12 240-259 105 105 105 160 160 160 F F F 13 260-279 106 106 106 161 161 161 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 108 108 108 167 167 167 F F F 17 340-359 106 106 106 162 162 162 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 104 104 104 159 159 159 F F F 20 400-419 107 107 107 163 163 163 F F F 21 420-439 107 107 107 163 163 163 F F F 22 440-459 108 108 108 165 165 165 F F F 23 460-479 106 106 106 161 161 161 F F F 24 480-499 108 108 108 167 167 167 T T T 25 500-519 106 106 106 162 162 162 T T T 26 520-539 104 104 104 157 157 157 F F F 27 540-559 104 104 104 157 157 157 F F F 28 560-579 108 108 108 167 167 167 F F F 29 580-599 108 108 108 167 167 167 F F F Max gen: 108, Max flush: 167 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 F F F 1 020-039 109 109 109 168 168 168 F F F 2 040-059 108 108 108 167 167 167 F F F 3 060-079 106 106 106 161 161 161 F F F 4 080-099 108 108 108 167 167 167 F F F 5 100-119 106 106 106 161 161 161 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 106 106 106 162 162 162 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 108 108 108 165 165 165 F F F 10 200-219 106 106 106 161 161 161 F F F 11 220-239 106 106 106 162 162 162 F F F 12 240-259 105 105 105 160 160 160 F F F 13 260-279 106 106 106 161 161 161 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 108 108 108 167 167 167 F F F 17 340-359 106 106 106 162 162 162 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 104 104 104 159 159 159 F F F 20 400-419 107 107 107 163 163 163 F F F 21 420-439 107 107 107 163 163 163 F F F 22 440-459 108 108 108 165 165 165 F F F 23 460-479 106 106 106 161 161 161 F F F 24 480-499 109 109 109 168 168 168 F F F 25 500-519 109 109 109 168 168 168 F F F 26 520-539 104 104 104 157 157 157 F F F 27 540-559 104 104 104 157 157 157 F F F 28 560-579 108 108 108 167 167 167 F F F 29 580-599 108 108 108 167 167 167 F F F Max gen: 109, Max flush: 168 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.40358629Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.447764469Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.448401862Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.448435259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.448448847Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.448468245Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632} {"msg":"Crucible 1386b731-cc8c-4b40-800e-e216ffed7f82 has session id: 5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.448964953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Upstairs opts: Upstairs UUID: 1386b731-cc8c-4b40-800e-e216ffed7f82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449004089Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Crucible stats registered with UUID: 1386b731-cc8c-4b40-800e-e216ffed7f82","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449027936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:18.449047434Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449485868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449516605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632} {"msg":"1386b731-cc8c-4b40-800e-e216ffed7f82 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449569079Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449594797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"0","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449623874Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"0","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449637852Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449651901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449664879Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"2","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449678998Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"2","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449699936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449722583Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449746271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:62982)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.449769228Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:18.44993957Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:18.449980356Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:18.450010043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:50313)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.450032471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.450150568Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.450484373Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"2","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.45051063Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"0","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.451106598Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"0","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.451134785Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"2","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.451783417Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:18.451814373Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.451166575Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:44857)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.451394601Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.452435332Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.452939099Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[0]R flush_numbers[0..12]: [150, 168, 167, 161, 167, 161, 161, 162, 162, 165, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453522698Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[0]R generation[0..12]: [96, 109, 108, 106, 108, 106, 106, 106, 106, 108, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453547345Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453567113Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[1]R flush_numbers[0..12]: [150, 168, 167, 161, 167, 161, 161, 162, 162, 165, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453584501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[1]R generation[0..12]: [96, 109, 108, 106, 108, 106, 106, 106, 106, 108, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453601079Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453617978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[2]R flush_numbers[0..12]: [150, 168, 167, 161, 167, 161, 161, 162, 162, 165, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453634546Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[2]R generation[0..12]: [96, 109, 108, 106, 108, 106, 106, 106, 106, 108, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453650954Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453667852Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.45368927Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453705779Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453721327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453737085Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"":"downstairs","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453752994Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453767482Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"0","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.45378338Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"1","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453799129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"client":"2","":"downstairs","session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:23:28.453815447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453839134Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"1386b731-cc8c-4b40-800e-e216ffed7f82 is now active with session: 5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453854763Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453869301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632,"session_id":"5dee8a2e-fa10-4a72-9cc7-e7e02013a5f3"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.45388407Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632} {"msg":"Activated sub_volume 1386b731-cc8c-4b40-800e-e216ffed7f82","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:28.453897978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 40960 data: 9 9 10 11 10 11 11 10 12 11 02/30 Write block 146 len 40960 data: 10 10 12 10 10 9 10 11 10 9 03/30 Write block 547 len 4096 data: 9 04/30 Read block 439 len 32768 05/30 Read block 275 len 28672 06/30 Read block 57 len 32768 07/30 Write block 420 len 4096 data: 11 08/30 Read block 154 len 36864 09/30 Write block 388 len 16384 data: 8 8 8 10 10/30 Read block 79 len 40960 11/30 Write block 556 len 40960 data: 12 13 15 12 18 18 15 12 11 8 12/30 Read block 118 len 32768 13/30 Write block 66 len 32768 data: 11 10 9 10 9 8 8 8 14/30 Write block 153 len 8192 data: 12 11 15/30 Read block 336 len 12288 16/30 Write block 425 len 40960 data: 8 8 9 10 13 16 15 14 13 12 17/30 Write block 8 len 4096 data: 7 18/30 Read block 265 len 4096 19/30 Read block 254 len 24576 20/30 Write block 55 len 8192 data: 12 12 21/30 Write block 24 len 28672 data: 5 6 6 4 6 8 11 22/30 Read block 7 len 12288 23/30 Write block 260 len 36864 data: 4 6 6 6 6 7 8 8 12 24/30 Read block 299 len 24576 25/30 Read block 444 len 20480 26/30 Write block 506 len 12288 data: 8 8 7 27/30 Read block 402 len 8192 28/30 Read block 587 len 12288 29/30 Write block 466 len 4096 data: 13 30/30 Read block 140 len 24576 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 29 30 30 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:28.474698363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 150 150 150 T T T 1 020-039 109 109 109 168 168 168 T T T 2 040-059 108 108 108 167 167 167 T T T 3 060-079 106 106 106 161 161 161 T T T 4 080-099 108 108 108 167 167 167 F F F 5 100-119 106 106 106 161 161 161 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 106 106 106 162 162 162 T T T 8 160-179 106 106 106 162 162 162 F F F 9 180-199 108 108 108 165 165 165 F F F 10 200-219 106 106 106 161 161 161 F F F 11 220-239 106 106 106 162 162 162 F F F 12 240-259 105 105 105 160 160 160 F F F 13 260-279 106 106 106 161 161 161 T T T 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 108 108 108 167 167 167 F F F 17 340-359 106 106 106 162 162 162 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 104 104 104 159 159 159 T T T 20 400-419 107 107 107 163 163 163 F F F 21 420-439 107 107 107 163 163 163 T T T 22 440-459 108 108 108 165 165 165 F F F 23 460-479 106 106 106 161 161 161 T T T 24 480-499 109 109 109 168 168 168 F F F 25 500-519 109 109 109 168 168 168 T T T 26 520-539 104 104 104 157 157 157 F F F 27 540-559 104 104 104 157 157 157 T T T 28 560-579 108 108 108 167 167 167 T T T 29 580-599 108 108 108 167 167 167 F F F Max gen: 109, Max flush: 168 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 169 169 169 F F F 1 020-039 110 110 110 169 169 169 F F F 2 040-059 110 110 110 169 169 169 F F F 3 060-079 110 110 110 169 169 169 F F F 4 080-099 108 108 108 167 167 167 F F F 5 100-119 106 106 106 161 161 161 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 110 110 110 169 169 169 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 108 108 108 165 165 165 F F F 10 200-219 106 106 106 161 161 161 F F F 11 220-239 106 106 106 162 162 162 F F F 12 240-259 105 105 105 160 160 160 F F F 13 260-279 110 110 110 169 169 169 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 108 108 108 167 167 167 F F F 17 340-359 106 106 106 162 162 162 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 110 110 110 169 169 169 F F F 20 400-419 107 107 107 163 163 163 F F F 21 420-439 110 110 110 169 169 169 F F F 22 440-459 108 108 108 165 165 165 F F F 23 460-479 110 110 110 169 169 169 F F F 24 480-499 109 109 109 168 168 168 F F F 25 500-519 110 110 110 169 169 169 F F F 26 520-539 104 104 104 157 157 157 F F F 27 540-559 110 110 110 169 169 169 F F F 28 560-579 110 110 110 169 169 169 F F F 29 580-599 108 108 108 167 167 167 F F F Max gen: 110, Max flush: 169 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.762421509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:42.805149549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.805512401Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.836926411Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.837496351Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.837535927Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.837552455Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.837569933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640} {"msg":"Crucible e04accce-4c79-43d5-8b43-0fbd8875713e has session id: c4397240-03f9-4c71-8e87-1d68e71c73a9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.83826623Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Upstairs opts: Upstairs UUID: e04accce-4c79-43d5-8b43-0fbd8875713e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838294387Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Crucible stats registered with UUID: e04accce-4c79-43d5-8b43-0fbd8875713e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838308726Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:42.838321574Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838798884Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838820762Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640} {"msg":"e04accce-4c79-43d5-8b43-0fbd8875713e active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838888755Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838905383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"0","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.83893352Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"0","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838947419Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"1","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838964777Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"1","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.838986275Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"2","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839012382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"2","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839037159Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839063247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839089904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35017)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839114041Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33980)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839138988Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62781)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839165026Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839922326Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"2","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.839964442Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"1","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.840099528Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"0","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.840690485Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"1","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.840718033Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"2","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.840733711Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"0","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841285973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.84131164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[0]R flush_numbers[0..12]: [169, 169, 169, 169, 167, 161, 161, 169, 162, 165, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841326259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[0]R generation[0..12]: [110, 110, 110, 110, 108, 106, 106, 110, 106, 108, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841340757Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841354936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[1]R flush_numbers[0..12]: [169, 169, 169, 169, 167, 161, 161, 169, 162, 165, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841368434Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[1]R generation[0..12]: [110, 110, 110, 110, 108, 106, 106, 110, 106, 108, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841381863Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841396202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[2]R flush_numbers[0..12]: [169, 169, 169, 169, 167, 161, 161, 169, 162, 165, 161, 162]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.8414096Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[2]R generation[0..12]: [110, 110, 110, 110, 108, 106, 106, 110, 106, 108, 106, 106]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841422689Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841436107Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Max found gen is 111","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841448566Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Generation requested: 112 >= found:111","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841461604Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841474263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841490161Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"":"downstairs","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.84150328Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841514999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"0","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841534537Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"1","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841547666Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"client":"2","":"downstairs","session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841560704Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841572733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"e04accce-4c79-43d5-8b43-0fbd8875713e is now active with session: c4397240-03f9-4c71-8e87-1d68e71c73a9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841585202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.84159676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640,"session_id":"c4397240-03f9-4c71-8e87-1d68e71c73a9"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841608589Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640} {"msg":"Activated sub_volume e04accce-4c79-43d5-8b43-0fbd8875713e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:42.841619458Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 243 len 40960 data: 11 9 9 7 9 9 7 7 8 8 02/30 Flush 03/30 Write block 527 len 36864 data: 8 8 7 7 6 7 8 10 9 04/30 Write block 29 len 40960 data: 9 12 12 12 11 11 11 10 11 9 05/30 Write block 562 len 8192 data: 16 13 06/30 Flush 07/30 Write block 222 len 24576 data: 13 14 12 11 8 8 08/30 Read block 397 len 20480 09/30 Write block 158 len 4096 data: 9 10/30 Read block 535 len 12288 11/30 Read block 519 len 36864 12/30 Read block 547 len 12288 13/30 Read block 8 len 4096 14/30 Read block 583 len 12288 15/30 Read block 175 len 12288 16/30 Write block 6 len 20480 data: 8 8 8 7 7 17/30 Read block 534 len 12288 18/30 Flush 19/30 Write block 469 len 12288 data: 14 16 16 20/30 Read block 491 len 32768 21/30 Read block 483 len 36864 22/30 Read block 101 len 16384 23/30 Read block 516 len 32768 24/30 Write block 563 len 32768 data: 14 12 9 9 7 8 9 5 25/30 Read block 90 len 16384 26/30 Read block 308 len 4096 27/30 Read block 525 len 24576 28/30 Write block 413 len 32768 data: 9 10 9 11 13 10 13 12 29/30 Read block 3 len 16384 30/30 Write block 100 len 28672 data: 14 13 14 13 11 7 6 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:42.862740689Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 172 172 172 F F F 1 020-039 112 112 112 171 171 171 F F F 2 040-059 110 110 110 169 169 169 F F F 3 060-079 110 110 110 169 169 169 F F F 4 080-099 108 108 108 167 167 167 F F F 5 100-119 106 106 106 161 161 161 T T T 6 120-139 106 106 106 161 161 161 F F F 7 140-159 112 112 112 172 172 172 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 108 108 108 165 165 165 F F F 10 200-219 106 106 106 161 161 161 F F F 11 220-239 112 112 112 172 172 172 F F F 12 240-259 112 112 112 170 170 170 F F F 13 260-279 110 110 110 169 169 169 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 108 108 108 167 167 167 F F F 17 340-359 106 106 106 162 162 162 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 110 110 110 169 169 169 F F F 20 400-419 107 107 107 163 163 163 T T T 21 420-439 110 110 110 169 169 169 T T T 22 440-459 108 108 108 165 165 165 F F F 23 460-479 110 110 110 169 169 169 T T T 24 480-499 109 109 109 168 168 168 F F F 25 500-519 110 110 110 169 169 169 F F F 26 520-539 112 112 112 171 171 171 F F F 27 540-559 110 110 110 169 169 169 F F F 28 560-579 112 112 112 171 171 171 T T T 29 580-599 108 108 108 167 167 167 F F F Max gen: 112, Max flush: 172 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 172 172 172 F F F 1 020-039 112 112 112 171 171 171 F F F 2 040-059 110 110 110 169 169 169 F F F 3 060-079 110 110 110 169 169 169 F F F 4 080-099 108 108 108 167 167 167 F F F 5 100-119 113 113 113 173 173 173 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 112 112 112 172 172 172 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 108 108 108 165 165 165 F F F 10 200-219 106 106 106 161 161 161 F F F 11 220-239 112 112 112 172 172 172 F F F 12 240-259 112 112 112 170 170 170 F F F 13 260-279 110 110 110 169 169 169 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 108 108 108 167 167 167 F F F 17 340-359 106 106 106 162 162 162 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 110 110 110 169 169 169 F F F 20 400-419 113 113 113 173 173 173 F F F 21 420-439 113 113 113 173 173 173 F F F 22 440-459 108 108 108 165 165 165 F F F 23 460-479 113 113 113 173 173 173 F F F 24 480-499 109 109 109 168 168 168 F F F 25 500-519 110 110 110 169 169 169 F F F 26 520-539 112 112 112 171 171 171 F F F 27 540-559 110 110 110 169 169 169 F F F 28 560-579 113 113 113 173 173 173 F F F 29 580-599 108 108 108 167 167 167 F F F Max gen: 113, Max flush: 173 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.129038114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:47.173374017Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.173770895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.204938421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.205528209Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.205563055Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.205577524Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.205591622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648} {"msg":"Crucible 5a996ed0-794a-4f53-8158-047d80ce271c has session id: 5c99a162-571a-4ee0-b8ad-691ca70844c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206138245Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Upstairs opts: Upstairs UUID: 5a996ed0-794a-4f53-8158-047d80ce271c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206171661Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Crucible stats registered with UUID: 5a996ed0-794a-4f53-8158-047d80ce271c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206192119Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:47.206204788Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206607396Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206628993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648} {"msg":"5a996ed0-794a-4f53-8158-047d80ce271c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206697196Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206713524Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"0","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206741042Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"0","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.20675484Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"1","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206769019Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"1","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206782207Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"2","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206796206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"2","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206816783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206842191Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206863159Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:33523)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.206882807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37185)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.207124701Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63735)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.207206362Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.207581023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"1","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.207713049Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"2","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.207942865Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"0","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.208107148Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"1","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.208237174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"2","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.208593797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"0","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.208739481Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.208826762Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[0]R flush_numbers[0..12]: [172, 171, 169, 169, 167, 173, 161, 172, 162, 165, 161, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209108743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[0]R generation[0..12]: [112, 112, 110, 110, 108, 113, 106, 112, 106, 108, 106, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.20913256Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209147509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[1]R flush_numbers[0..12]: [172, 171, 169, 169, 167, 173, 161, 172, 162, 165, 161, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209161197Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[1]R generation[0..12]: [112, 112, 110, 110, 108, 113, 106, 112, 106, 108, 106, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209178905Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209192994Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[2]R flush_numbers[0..12]: [172, 171, 169, 169, 167, 173, 161, 172, 162, 165, 161, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209206403Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[2]R generation[0..12]: [112, 112, 110, 110, 108, 113, 106, 112, 106, 108, 106, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209219621Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.20923348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209246148Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:23:47.209258857Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209277685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209290104Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"":"downstairs","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209303022Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209314741Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"0","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.20932785Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"1","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209340688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"client":"2","":"downstairs","session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209353447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209367195Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"5a996ed0-794a-4f53-8158-047d80ce271c is now active with session: 5c99a162-571a-4ee0-b8ad-691ca70844c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209379784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209391473Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648,"session_id":"5c99a162-571a-4ee0-b8ad-691ca70844c2"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.209403372Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Activated sub_volume 5a996ed0-794a-4f53-8158-047d80ce271c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:47.20941803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 4 len 20480 data: 6 6 9 9 9 03/30 Flush 04/30 Write block 506 len 16384 data: 9 9 8 10 05/30 Read block 323 len 32768 06/30 Flush 07/30 Write block 338 len 24576 data: 11 11 12 13 10 9 08/30 Write block 575 len 32768 data: 8 8 8 9 8 7 11 10 09/30 Read block 513 len 20480 10/30 Read block 188 len 40960 11/30 Write block 491 len 16384 data: 5 7 8 9 12/30 Read block 432 len 40960 13/30 Flush 14/30 Write block 463 len 36864 data: 13 12 10 14 14 14 15 17 17 15/30 Write block 395 len 24576 data: 13 14 12 13 14 13 16/30 Read block 117 len 20480 17/30 Read block 138 len 8192 18/30 Write block 67 len 20480 data: 11 10 11 10 9 19/30 Read block 147 len 8192 20/30 Read block 437 len 16384 21/30 Read block 46 len 4096 22/30 Read block 151 len 40960 23/30 Write block 468 len 28672 data: 15 16 18 18 14 14 10 24/30 Write block 517 len 40960 data: 11 11 10 10 10 10 8 7 8 9 25/30 Read block 551 len 20480 26/30 Read block 322 len 16384 27/30 Read block 548 len 36864 28/30 Write block 199 len 28672 data: 7 10 9 8 8 5 7 29/30 Write block 194 len 20480 data: 10 9 10 8 7 30/30 Read block 249 len 36864 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Write 6 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 15 17 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) Some(JobId(1012)) Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:47.230760978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 175 175 175 F F F 1 020-039 112 112 112 171 171 171 F F F 2 040-059 110 110 110 169 169 169 F F F 3 060-079 110 110 110 169 169 169 T T T 4 080-099 108 108 108 167 167 167 F F F 5 100-119 113 113 113 173 173 173 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 112 112 112 172 172 172 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 108 108 108 165 165 165 T T T 10 200-219 106 106 106 161 161 161 T T T 11 220-239 112 112 112 172 172 172 F F F 12 240-259 112 112 112 170 170 170 F F F 13 260-279 110 110 110 169 169 169 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 114 114 114 177 177 177 F F F 17 340-359 114 114 114 177 177 177 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 110 110 110 169 169 169 T T T 20 400-419 113 113 113 173 173 173 T T T 21 420-439 113 113 113 173 173 173 F F F 22 440-459 108 108 108 165 165 165 F F F 23 460-479 113 113 113 173 173 173 T T T 24 480-499 114 114 114 177 177 177 F F F 25 500-519 114 114 114 176 176 176 T T T 26 520-539 112 112 112 171 171 171 T T T 27 540-559 110 110 110 169 169 169 F F F 28 560-579 114 114 114 177 177 177 F F F 29 580-599 114 114 114 177 177 177 F F F Max gen: 114, Max flush: 177 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 175 175 175 F F F 1 020-039 112 112 112 171 171 171 F F F 2 040-059 110 110 110 169 169 169 F F F 3 060-079 115 115 115 178 178 178 F F F 4 080-099 108 108 108 167 167 167 F F F 5 100-119 113 113 113 173 173 173 F F F 6 120-139 106 106 106 161 161 161 F F F 7 140-159 112 112 112 172 172 172 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 115 115 115 178 178 178 F F F 10 200-219 115 115 115 178 178 178 F F F 11 220-239 112 112 112 172 172 172 F F F 12 240-259 112 112 112 170 170 170 F F F 13 260-279 110 110 110 169 169 169 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 114 114 114 177 177 177 F F F 17 340-359 114 114 114 177 177 177 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 115 115 115 178 178 178 F F F 20 400-419 115 115 115 178 178 178 F F F 21 420-439 113 113 113 173 173 173 F F F 22 440-459 108 108 108 165 165 165 F F F 23 460-479 115 115 115 178 178 178 F F F 24 480-499 114 114 114 177 177 177 F F F 25 500-519 115 115 115 178 178 178 F F F 26 520-539 115 115 115 178 178 178 F F F 27 540-559 110 110 110 169 169 169 F F F 28 560-579 114 114 114 177 177 177 F F F 29 580-599 114 114 114 177 177 177 F F F Max gen: 115, Max flush: 178 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.511329233Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.555894521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.556500748Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.556534844Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.556550162Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.556565001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656} {"msg":"Crucible 5f83615c-b8be-4dd7-af22-26c78c537808 has session id: 9eeec47b-6d00-46f4-822c-5758ec939e66","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.557246999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Upstairs opts: Upstairs UUID: 5f83615c-b8be-4dd7-af22-26c78c537808, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.557285235Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Crucible stats registered with UUID: 5f83615c-b8be-4dd7-af22-26c78c537808","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.557308753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:51.557329261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.557893771Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.557933017Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656} {"msg":"5f83615c-b8be-4dd7-af22-26c78c537808 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.5579989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558027917Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"0","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558059464Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"0","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558078632Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"1","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.55809357Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"1","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558116928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"2","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558143215Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"2","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558166953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.55819399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558220087Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"ds_connection connected from Ok(127.0.0.1:46451)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558244735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"ds_connection connected from Ok(127.0.0.1:49155)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558269582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"ds_connection connected from Ok(127.0.0.1:55731)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.558394799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.55895726Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"0","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.559001735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"2","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.559612241Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"2","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.559760785Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"0","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.560244445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.560351493Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.560448543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"1","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.560960529Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"1","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[0]R flush_numbers[0..12]: [175, 171, 169, 178, 167, 173, 161, 172, 162, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561472445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[0]R generation[0..12]: [114, 112, 110, 115, 108, 113, 106, 112, 106, 115, 115, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561493273Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561507902Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[1]R flush_numbers[0..12]: [175, 171, 169, 178, 167, 173, 161, 172, 162, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.56152158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[1]R generation[0..12]: [114, 112, 110, 115, 108, 113, 106, 112, 106, 115, 115, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561534919Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561548588Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[2]R flush_numbers[0..12]: [175, 171, 169, 178, 167, 173, 161, 172, 162, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561561986Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[2]R generation[0..12]: [114, 112, 110, 115, 108, 113, 106, 112, 106, 115, 115, 112]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561575065Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561588293Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561600902Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.56161866Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561632019Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","Wait for a query_work_queue command to finish before sending IO pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561652746Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"":"downstairs","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561665965Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561677714Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"0","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561690733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"1","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561703761Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"client":"2","":"downstairs","session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.56171665Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561728678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"5f83615c-b8be-4dd7-af22-26c78c537808 is now active with session: 9eeec47b-6d00-46f4-822c-5758ec939e66","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561741197Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561752836Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656,"session_id":"9eeec47b-6d00-46f4-822c-5758ec939e66"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561764715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656} {"msg":"Activated sub_volume 5f83615c-b8be-4dd7-af22-26c78c537808","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:51.561775533Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 125 len 28672 data: 7 6 5 8 7 8 7 02/30 Read block 281 len 8192 03/30 Write block 65 len 16384 data: 11 12 12 11 04/30 Read block 491 len 24576 05/30 Flush 06/30 Flush 07/30 Flush 08/30 Write block 337 len 24576 data: 9 12 12 13 14 11 09/30 Write block 39 len 32768 data: 7 8 11 10 10 10 10 11 10/30 Write block 79 len 32768 data: 6 7 6 7 6 6 6 6 11/30 Flush 12/30 Write block 101 len 32768 data: 14 15 14 12 8 7 6 7 13/30 Write block 427 len 32768 data: 10 11 14 17 16 15 14 13 14/30 Read block 401 len 4096 15/30 Read block 124 len 32768 16/30 Read block 9 len 24576 17/30 Write block 220 len 4096 data: 17 18/30 Write block 390 len 20480 data: 9 11 12 12 12 19/30 Write block 252 len 20480 data: 9 7 8 9 6 20/30 Write block 599 len 4096 data: 3 21/30 Write block 2 len 8192 data: 6 5 22/30 Flush 23/30 Write block 63 len 36864 data: 11 11 12 13 13 12 12 11 10 24/30 Read block 448 len 12288 25/30 Write block 15 len 16384 data: 9 9 9 8 26/30 Read block 210 len 16384 27/30 Write block 270 len 32768 data: 12 12 13 10 9 9 6 7 28/30 Read block 197 len 16384 29/30 Write block 502 len 28672 data: 7 7 8 9 10 10 9 30/30 Read block 307 len 20480 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 7 8 7 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:51.582173711Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 116 116 116 183 183 183 T T T 1 020-039 116 116 116 182 182 182 F F F 2 040-059 116 116 116 182 182 182 F F F 3 060-079 116 116 116 182 182 182 T T T 4 080-099 116 116 116 182 182 182 F F F 5 100-119 116 116 116 183 183 183 F F F 6 120-139 116 116 116 179 179 179 F F F 7 140-159 112 112 112 172 172 172 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 115 115 115 178 178 178 F F F 10 200-219 115 115 115 178 178 178 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 116 116 116 183 183 183 F F F 13 260-279 110 110 110 169 169 169 T T T 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 116 116 116 182 182 182 F F F 17 340-359 116 116 116 182 182 182 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 116 116 116 183 183 183 F F F 20 400-419 115 115 115 178 178 178 F F F 21 420-439 116 116 116 183 183 183 F F F 22 440-459 108 108 108 165 165 165 F F F 23 460-479 115 115 115 178 178 178 F F F 24 480-499 114 114 114 177 177 177 F F F 25 500-519 115 115 115 178 178 178 T T T 26 520-539 115 115 115 178 178 178 F F F 27 540-559 110 110 110 169 169 169 F F F 28 560-579 114 114 114 177 177 177 F F F 29 580-599 116 116 116 183 183 183 F F F Max gen: 116, Max flush: 183 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 117 117 117 184 184 184 F F F 1 020-039 116 116 116 182 182 182 F F F 2 040-059 116 116 116 182 182 182 F F F 3 060-079 117 117 117 184 184 184 F F F 4 080-099 116 116 116 182 182 182 F F F 5 100-119 116 116 116 183 183 183 F F F 6 120-139 116 116 116 179 179 179 F F F 7 140-159 112 112 112 172 172 172 F F F 8 160-179 106 106 106 162 162 162 F F F 9 180-199 115 115 115 178 178 178 F F F 10 200-219 115 115 115 178 178 178 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 116 116 116 183 183 183 F F F 13 260-279 117 117 117 184 184 184 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 116 116 116 182 182 182 F F F 17 340-359 116 116 116 182 182 182 F F F 18 360-379 98 98 98 153 153 153 F F F 19 380-399 116 116 116 183 183 183 F F F 20 400-419 115 115 115 178 178 178 F F F 21 420-439 116 116 116 183 183 183 F F F 22 440-459 108 108 108 165 165 165 F F F 23 460-479 115 115 115 178 178 178 F F F 24 480-499 114 114 114 177 177 177 F F F 25 500-519 117 117 117 184 184 184 F F F 26 520-539 115 115 115 178 178 178 F F F 27 540-559 110 110 110 169 169 169 F F F 28 560-579 114 114 114 177 177 177 F F F 29 580-599 116 116 116 183 183 183 F F F Max gen: 117, Max flush: 184 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.813545843Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:55.855893045Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.856340618Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.887657617Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.888234167Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.888274403Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.888291671Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.888309379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664} {"msg":"Crucible c4a13139-2605-47a3-9882-0764f40b7d59 has session id: 93a55436-aabb-4540-af65-1f5f4f9119c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.88896781Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Upstairs opts: Upstairs UUID: c4a13139-2605-47a3-9882-0764f40b7d59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889002006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Crucible stats registered with UUID: c4a13139-2605-47a3-9882-0764f40b7d59","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889018854Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:55.889031873Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889483216Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889506063Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664} {"msg":"c4a13139-2605-47a3-9882-0764f40b7d59 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889578016Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889594844Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"0","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889621951Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"0","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889637809Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"1","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889662647Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"1","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889686794Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"2","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889736989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"2","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889775225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889805462Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889833069Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:46682)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889867985Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:35761)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.889895752Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48023)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.88992127Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.890720646Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"1","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.89076973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"2","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.890807877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"0","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.891420422Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"1","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.891462438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"2","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.891493394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"0","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892069334Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892173083Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[0]R flush_numbers[0..12]: [184, 182, 182, 184, 182, 183, 179, 172, 162, 178, 178, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892236287Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[0]R generation[0..12]: [117, 116, 116, 117, 116, 116, 116, 112, 106, 115, 115, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892256244Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892270803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[1]R flush_numbers[0..12]: [184, 182, 182, 184, 182, 183, 179, 172, 162, 178, 178, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892287861Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[1]R generation[0..12]: [117, 116, 116, 117, 116, 116, 116, 112, 106, 115, 115, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.89230157Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892315568Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[2]R flush_numbers[0..12]: [184, 182, 182, 184, 182, 183, 179, 172, 162, 178, 178, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892329597Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[2]R generation[0..12]: [117, 116, 116, 117, 116, 116, 116, 112, 106, 115, 115, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892342915Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892356494Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892378152Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892404519Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892430746Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892446464Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"":"downstairs","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892466452Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892493369Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"0","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892518397Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"1","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892538075Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"client":"2","":"downstairs","session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892554673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892575111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"c4a13139-2605-47a3-9882-0764f40b7d59 is now active with session: 93a55436-aabb-4540-af65-1f5f4f9119c2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892598818Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892620666Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664,"session_id":"93a55436-aabb-4540-af65-1f5f4f9119c2"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892634005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664} {"msg":"Activated sub_volume c4a13139-2605-47a3-9882-0764f40b7d59","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:23:55.892647193Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 354 len 8192 data: 8 7 03/30 Write block 135 len 16384 data: 8 9 10 10 04/30 Read block 138 len 8192 05/30 Read block 276 len 32768 06/30 Write block 327 len 36864 data: 7 7 7 10 9 8 9 9 10 07/30 Read block 561 len 40960 08/30 Write block 523 len 28672 data: 9 8 9 10 9 9 8 09/30 Read block 67 len 4096 10/30 Read block 179 len 28672 11/30 Read block 460 len 36864 12/30 Read block 14 len 20480 13/30 Read block 123 len 20480 14/30 Write block 354 len 28672 data: 9 8 7 7 6 7 6 15/30 Write block 552 len 20480 data: 13 13 11 13 13 16/30 Flush 17/30 Write block 126 len 28672 data: 7 6 9 8 9 8 9 18/30 Write block 446 len 36864 data: 9 9 8 10 10 11 10 9 12 19/30 Read block 333 len 4096 20/30 Read block 587 len 4096 21/30 Flush 22/30 Write block 6 len 40960 data: 10 10 10 8 8 8 10 8 8 10 23/30 Read block 170 len 32768 24/30 Read block 84 len 4096 25/30 Write block 543 len 28672 data: 11 10 9 10 10 10 11 26/30 Read block 481 len 32768 27/30 Write block 158 len 36864 data: 10 9 10 9 11 14 12 10 11 28/30 Write block 444 len 12288 data: 8 7 10 29/30 Write block 458 len 16384 data: 11 13 13 14 30/30 Read block 373 len 24576 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 0 2 Done 9 7 9 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:23:55.914073313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 117 117 117 184 184 184 T T T 1 020-039 116 116 116 182 182 182 F F F 2 040-059 116 116 116 182 182 182 F F F 3 060-079 117 117 117 184 184 184 F F F 4 080-099 116 116 116 182 182 182 F F F 5 100-119 116 116 116 183 183 183 F F F 6 120-139 118 118 118 187 187 187 F F F 7 140-159 112 112 112 172 172 172 T T T 8 160-179 106 106 106 162 162 162 T T T 9 180-199 115 115 115 178 178 178 F F F 10 200-219 115 115 115 178 178 178 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 116 116 116 183 183 183 F F F 13 260-279 117 117 117 184 184 184 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 118 118 118 186 186 186 F F F 17 340-359 118 118 118 186 186 186 F F F 18 360-379 118 118 118 186 186 186 F F F 19 380-399 116 116 116 183 183 183 F F F 20 400-419 115 115 115 178 178 178 F F F 21 420-439 116 116 116 183 183 183 F F F 22 440-459 118 118 118 187 187 187 T T T 23 460-479 115 115 115 178 178 178 T T T 24 480-499 114 114 114 177 177 177 F F F 25 500-519 117 117 117 184 184 184 F F F 26 520-539 118 118 118 186 186 186 F F F 27 540-559 118 118 118 186 186 186 T T T 28 560-579 114 114 114 177 177 177 F F F 29 580-599 116 116 116 183 183 183 F F F Max gen: 118, Max flush: 187 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 188 188 188 F F F 1 020-039 116 116 116 182 182 182 F F F 2 040-059 116 116 116 182 182 182 F F F 3 060-079 117 117 117 184 184 184 F F F 4 080-099 116 116 116 182 182 182 F F F 5 100-119 116 116 116 183 183 183 F F F 6 120-139 118 118 118 187 187 187 F F F 7 140-159 119 119 119 188 188 188 F F F 8 160-179 119 119 119 188 188 188 F F F 9 180-199 115 115 115 178 178 178 F F F 10 200-219 115 115 115 178 178 178 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 116 116 116 183 183 183 F F F 13 260-279 117 117 117 184 184 184 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 118 118 118 186 186 186 F F F 17 340-359 118 118 118 186 186 186 F F F 18 360-379 118 118 118 186 186 186 F F F 19 380-399 116 116 116 183 183 183 F F F 20 400-419 115 115 115 178 178 178 F F F 21 420-439 116 116 116 183 183 183 F F F 22 440-459 119 119 119 188 188 188 F F F 23 460-479 119 119 119 188 188 188 F F F 24 480-499 114 114 114 177 177 177 F F F 25 500-519 117 117 117 184 184 184 F F F 26 520-539 118 118 118 186 186 186 F F F 27 540-559 119 119 119 188 188 188 F F F 28 560-579 114 114 114 177 177 177 F F F 29 580-599 116 116 116 183 183 183 F F F Max gen: 119, Max flush: 188 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.187233877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.230693951Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.231442673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.231481028Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.231497517Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.231514665Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672} {"msg":"Crucible 59a710ca-3a49-4b20-a8fb-4a3830c3850c has session id: b8dc90a7-d536-4ebe-87c8-5c0ec535f83e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.231994904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Upstairs opts: Upstairs UUID: 59a710ca-3a49-4b20-a8fb-4a3830c3850c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232073716Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Crucible stats registered with UUID: 59a710ca-3a49-4b20-a8fb-4a3830c3850c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232103083Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:00.232124231Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232647786Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232681202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672} {"msg":"59a710ca-3a49-4b20-a8fb-4a3830c3850c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232750785Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232769113Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"0","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.23279572Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"0","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232813558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232827957Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232840995Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"2","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232854554Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"2","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232867203Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.232880281Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.23289337Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62748)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.233048264Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44254)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.23308872Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:00.233120486Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:00.233150663Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:00.233247713Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.233377909Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.233704245Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"2","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.233862228Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"0","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.234249798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"2","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.234385153Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"0","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.234856344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:00.234965472Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.234808131Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41077)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.234927258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.235872519Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.236451178Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[0]R flush_numbers[0..12]: [188, 182, 182, 184, 182, 183, 187, 188, 188, 178, 178, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237061924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[0]R generation[0..12]: [119, 116, 116, 117, 116, 116, 118, 119, 119, 115, 115, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237092571Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237111898Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[1]R flush_numbers[0..12]: [188, 182, 182, 184, 182, 183, 187, 188, 188, 178, 178, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237129437Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[1]R generation[0..12]: [119, 116, 116, 117, 116, 116, 118, 119, 119, 115, 115, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237147295Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237171572Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[2]R flush_numbers[0..12]: [188, 182, 182, 184, 182, 183, 187, 188, 188, 178, 178, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.2371904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[2]R generation[0..12]: [119, 116, 116, 117, 116, 116, 118, 119, 119, 115, 115, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237208459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237225367Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237240975Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237256973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237272752Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.23728837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"":"downstairs","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237304228Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237318737Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"0","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237334795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"1","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:24:10.237350693Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"client":"2","":"downstairs","session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237372851Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.23738784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"59a710ca-3a49-4b20-a8fb-4a3830c3850c is now active with session: b8dc90a7-d536-4ebe-87c8-5c0ec535f83e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237404508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237423176Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672,"session_id":"b8dc90a7-d536-4ebe-87c8-5c0ec535f83e"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237439634Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672} {"msg":"Activated sub_volume 59a710ca-3a49-4b20-a8fb-4a3830c3850c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:10.237454163Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 96 len 20480 data: 11 10 10 14 15 02/30 Read block 424 len 12288 03/30 Read block 239 len 24576 04/30 Read block 38 len 32768 05/30 Flush 06/30 Write block 402 len 12288 data: 14 15 16 07/30 Write block 134 len 28672 data: 10 9 10 11 11 12 11 08/30 Read block 438 len 20480 09/30 Write block 335 len 20480 data: 11 9 10 13 13 10/30 Read block 222 len 12288 11/30 Read block 105 len 40960 12/30 Read block 259 len 20480 13/30 Read block 264 len 32768 14/30 Write block 427 len 32768 data: 11 12 15 18 17 16 15 14 15/30 Read block 335 len 12288 16/30 Read block 302 len 16384 17/30 Write block 366 len 32768 data: 8 8 7 8 8 8 9 9 18/30 Read block 478 len 16384 19/30 Flush 20/30 Write block 196 len 40960 data: 11 9 8 8 11 10 9 9 6 8 21/30 Flush 22/30 Write block 252 len 20480 data: 10 8 9 10 7 23/30 Read block 485 len 40960 24/30 Read block 565 len 4096 25/30 Read block 212 len 36864 26/30 Flush 27/30 Write block 141 len 20480 data: 12 13 13 12 10 28/30 Write block 528 len 32768 data: 10 9 8 7 8 9 11 10 29/30 Read block 163 len 12288 30/30 Write block 81 len 36864 data: 7 8 7 7 7 7 7 8 7 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:10.259491218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1672} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 188 188 188 F F F 1 020-039 116 116 116 182 182 182 F F F 2 040-059 116 116 116 182 182 182 F F F 3 060-079 117 117 117 184 184 184 F F F 4 080-099 120 120 120 189 189 189 T T T 5 100-119 120 120 120 189 189 189 F F F 6 120-139 120 120 120 190 190 190 F F F 7 140-159 120 120 120 190 190 190 T T T 8 160-179 119 119 119 188 188 188 F F F 9 180-199 120 120 120 191 191 191 F F F 10 200-219 120 120 120 191 191 191 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 120 120 120 192 192 192 F F F 13 260-279 117 117 117 184 184 184 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 120 120 120 190 190 190 F F F 17 340-359 118 118 118 186 186 186 F F F 18 360-379 120 120 120 190 190 190 F F F 19 380-399 116 116 116 183 183 183 F F F 20 400-419 120 120 120 190 190 190 F F F 21 420-439 120 120 120 190 190 190 F F F 22 440-459 119 119 119 188 188 188 F F F 23 460-479 119 119 119 188 188 188 F F F 24 480-499 114 114 114 177 177 177 F F F 25 500-519 117 117 117 184 184 184 F F F 26 520-539 118 118 118 186 186 186 T T T 27 540-559 119 119 119 188 188 188 F F F 28 560-579 114 114 114 177 177 177 F F F 29 580-599 116 116 116 183 183 183 F F F Max gen: 120, Max flush: 192 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 188 188 188 F F F 1 020-039 116 116 116 182 182 182 F F F 2 040-059 116 116 116 182 182 182 F F F 3 060-079 117 117 117 184 184 184 F F F 4 080-099 121 121 121 193 193 193 F F F 5 100-119 120 120 120 189 189 189 F F F 6 120-139 120 120 120 190 190 190 F F F 7 140-159 121 121 121 193 193 193 F F F 8 160-179 119 119 119 188 188 188 F F F 9 180-199 120 120 120 191 191 191 F F F 10 200-219 120 120 120 191 191 191 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 120 120 120 192 192 192 F F F 13 260-279 117 117 117 184 184 184 F F F 14 280-299 92 92 92 144 144 144 F F F 15 300-319 104 104 104 158 158 158 F F F 16 320-339 120 120 120 190 190 190 F F F 17 340-359 118 118 118 186 186 186 F F F 18 360-379 120 120 120 190 190 190 F F F 19 380-399 116 116 116 183 183 183 F F F 20 400-419 120 120 120 190 190 190 F F F 21 420-439 120 120 120 190 190 190 F F F 22 440-459 119 119 119 188 188 188 F F F 23 460-479 119 119 119 188 188 188 F F F 24 480-499 114 114 114 177 177 177 F F F 25 500-519 117 117 117 184 184 184 F F F 26 520-539 121 121 121 193 193 193 F F F 27 540-559 119 119 119 188 188 188 F F F 28 560-579 114 114 114 177 177 177 F F F 29 580-599 116 116 116 183 183 183 F F F Max gen: 121, Max flush: 193 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.489919212Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.53410458Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.534745503Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.534787868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.534812466Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.534831114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680} {"msg":"Crucible bf428d84-4e5e-4a3a-bdd4-8c41ae5f36a4 has session id: d073f05f-ce28-413c-b323-ceb50b2ff79f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.53534774Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Upstairs opts: Upstairs UUID: bf428d84-4e5e-4a3a-bdd4-8c41ae5f36a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.535388075Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Crucible stats registered with UUID: bf428d84-4e5e-4a3a-bdd4-8c41ae5f36a4","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.535411333Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:14.535433041Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.535824139Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.535847837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680} {"msg":"bf428d84-4e5e-4a3a-bdd4-8c41ae5f36a4 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.535932408Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.535950586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"0","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.535977903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"0","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.535991712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"1","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.53600674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"1","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.536019949Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"2","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.536034837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"2","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.536146946Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.536174213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.53619869Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48418)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.536237826Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60255)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.536264603Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48539)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.53629474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.536921784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"0","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.53696287Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"1","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.537443509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"1","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.537572806Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"0","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.537948407Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.53820117Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.538893697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"2","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.539474166Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"2","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[0]R flush_numbers[0..12]: [188, 182, 182, 184, 193, 189, 190, 193, 188, 191, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.54001174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[0]R generation[0..12]: [119, 116, 116, 117, 121, 120, 120, 121, 119, 120, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540039787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540055585Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[1]R flush_numbers[0..12]: [188, 182, 182, 184, 193, 189, 190, 193, 188, 191, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540070784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[1]R generation[0..12]: [119, 116, 116, 117, 121, 120, 120, 121, 119, 120, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540084472Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.54009821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[2]R flush_numbers[0..12]: [188, 182, 182, 184, 193, 189, 190, 193, 188, 191, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540111719Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[2]R generation[0..12]: [119, 116, 116, 117, 121, 120, 120, 121, 119, 120, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540125388Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540138996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540151695Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540164444Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:24:14.540176902Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.54019526Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"":"downstairs","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540207889Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540231956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"0","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540247245Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"1","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540260153Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"client":"2","":"downstairs","session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540272952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540284901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"bf428d84-4e5e-4a3a-bdd4-8c41ae5f36a4 is now active with session: d073f05f-ce28-413c-b323-ceb50b2ff79f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.54029734Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540308919Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680,"session_id":"d073f05f-ce28-413c-b323-ceb50b2ff79f"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540320727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680} {"msg":"Activated sub_volume bf428d84-4e5e-4a3a-bdd4-8c41ae5f36a4","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:14.540331556Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 541 len 28672 data: 7 9 12 11 10 11 11 02/30 Read block 423 len 4096 03/30 Write block 143 len 16384 data: 14 13 11 11 04/30 Write block 518 len 16384 data: 12 11 11 11 05/30 Write block 42 len 36864 data: 11 11 11 11 12 12 11 12 12 06/30 Read block 423 len 8192 07/30 Read block 397 len 20480 08/30 Write block 504 len 40960 data: 9 10 11 11 10 11 13 15 12 11 09/30 Write block 451 len 36864 data: 12 11 10 13 13 15 13 12 14 10/30 Write block 578 len 28672 data: 10 9 8 12 11 10 13 11/30 Write block 447 len 20480 data: 10 9 11 11 13 12/30 Read block 59 len 4096 13/30 Write block 378 len 12288 data: 11 11 10 14/30 Write block 423 len 40960 data: 8 6 9 9 12 13 16 19 18 17 15/30 Write block 149 len 12288 data: 11 11 10 16/30 Write block 20 len 32768 data: 7 7 7 7 6 7 7 5 17/30 Read block 17 len 16384 18/30 Read block 489 len 32768 19/30 Read block 61 len 4096 20/30 Flush 21/30 Write block 458 len 16384 data: 13 15 14 15 22/30 Write block 108 len 32768 data: 8 7 7 7 9 7 8 6 23/30 Read block 283 len 36864 24/30 Write block 480 len 32768 data: 8 10 10 9 10 12 13 12 25/30 Write block 316 len 32768 data: 9 9 9 10 8 10 7 7 26/30 Write block 177 len 28672 data: 7 5 6 8 9 9 11 27/30 Write block 279 len 32768 data: 8 8 10 12 13 14 13 12 28/30 Write block 118 len 32768 data: 7 8 7 9 9 10 9 8 29/30 Write block 17 len 4096 data: 10 30/30 Read block 330 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 8 10 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:14.559692422Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1680} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 188 188 188 T T T 1 020-039 122 122 122 194 194 194 F F F 2 040-059 122 122 122 194 194 194 F F F 3 060-079 117 117 117 184 184 184 F F F 4 080-099 121 121 121 193 193 193 F F F 5 100-119 120 120 120 189 189 189 T T T 6 120-139 120 120 120 190 190 190 T T T 7 140-159 122 122 122 194 194 194 F F F 8 160-179 119 119 119 188 188 188 T T T 9 180-199 120 120 120 191 191 191 T T T 10 200-219 120 120 120 191 191 191 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 120 120 120 192 192 192 F F F 13 260-279 117 117 117 184 184 184 T T T 14 280-299 92 92 92 144 144 144 T T T 15 300-319 104 104 104 158 158 158 T T T 16 320-339 120 120 120 190 190 190 T T T 17 340-359 118 118 118 186 186 186 F F F 18 360-379 122 122 122 194 194 194 F F F 19 380-399 122 122 122 194 194 194 F F F 20 400-419 120 120 120 190 190 190 F F F 21 420-439 122 122 122 194 194 194 F F F 22 440-459 122 122 122 194 194 194 T T T 23 460-479 119 119 119 188 188 188 T T T 24 480-499 114 114 114 177 177 177 T T T 25 500-519 122 122 122 194 194 194 F F F 26 520-539 122 122 122 194 194 194 F F F 27 540-559 122 122 122 194 194 194 F F F 28 560-579 122 122 122 194 194 194 F F F 29 580-599 122 122 122 194 194 194 F F F Max gen: 122, Max flush: 194 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 122 122 122 194 194 194 F F F 3 060-079 117 117 117 184 184 184 F F F 4 080-099 121 121 121 193 193 193 F F F 5 100-119 123 123 123 195 195 195 F F F 6 120-139 123 123 123 195 195 195 F F F 7 140-159 122 122 122 194 194 194 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 123 123 123 195 195 195 F F F 10 200-219 120 120 120 191 191 191 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 120 120 120 192 192 192 F F F 13 260-279 123 123 123 195 195 195 F F F 14 280-299 123 123 123 195 195 195 F F F 15 300-319 123 123 123 195 195 195 F F F 16 320-339 123 123 123 195 195 195 F F F 17 340-359 118 118 118 186 186 186 F F F 18 360-379 122 122 122 194 194 194 F F F 19 380-399 122 122 122 194 194 194 F F F 20 400-419 120 120 120 190 190 190 F F F 21 420-439 122 122 122 194 194 194 F F F 22 440-459 123 123 123 195 195 195 F F F 23 460-479 123 123 123 195 195 195 F F F 24 480-499 123 123 123 195 195 195 F F F 25 500-519 122 122 122 194 194 194 F F F 26 520-539 122 122 122 194 194 194 F F F 27 540-559 122 122 122 194 194 194 F F F 28 560-579 122 122 122 194 194 194 F F F 29 580-599 122 122 122 194 194 194 F F F Max gen: 123, Max flush: 195 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.83140404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.875770419Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.876390634Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.87642415Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.876438039Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.876452518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688} {"msg":"Crucible 210c7afe-d57f-4297-bc49-407481d7271c has session id: 2bcb4359-1091-46bb-b3f0-90f3de0cf7e8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.876964524Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Upstairs opts: Upstairs UUID: 210c7afe-d57f-4297-bc49-407481d7271c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.87700316Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Crucible stats registered with UUID: 210c7afe-d57f-4297-bc49-407481d7271c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877026247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:28.877048645Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877580739Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877608676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688} {"msg":"210c7afe-d57f-4297-bc49-407481d7271c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877673499Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877691647Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"0","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877719014Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"0","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877732783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"1","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877749601Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"1","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877773319Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"2","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877841772Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"2","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.877879668Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.878113713Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.87814563Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56810)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.878167317Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47062)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.878187755Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45884)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.878213933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.878612271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"2","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.878657716Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"0","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.879467141Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"2","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.879506947Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"0","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.879994295Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"1","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.880195944Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.880218432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.880581744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"1","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[0]R flush_numbers[0..12]: [195, 194, 194, 184, 193, 195, 195, 194, 195, 195, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881086331Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[0]R generation[0..12]: [123, 122, 122, 117, 121, 123, 123, 122, 123, 123, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881106738Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881121367Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[1]R flush_numbers[0..12]: [195, 194, 194, 184, 193, 195, 195, 194, 195, 195, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881134986Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[1]R generation[0..12]: [123, 122, 122, 117, 121, 123, 123, 122, 123, 123, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881148424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881162303Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[2]R flush_numbers[0..12]: [195, 194, 194, 184, 193, 195, 195, 194, 195, 195, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881175961Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[2]R generation[0..12]: [123, 122, 122, 117, 121, 123, 123, 122, 123, 123, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881192589Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881206528Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881218967Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881231735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881244304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881262562Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"":"downstairs","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881275721Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881287539Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"0","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881300668Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"1","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881313847Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"client":"2","":"downstairs","session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881327255Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881339384Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"210c7afe-d57f-4297-bc49-407481d7271c is now active with session: 2bcb4359-1091-46bb-b3f0-90f3de0cf7e8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881351813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881363382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688,"session_id":"2bcb4359-1091-46bb-b3f0-90f3de0cf7e8"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.88137521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688} {"msg":"Activated sub_volume 210c7afe-d57f-4297-bc49-407481d7271c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:28.881386309Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 593 len 12288 data: 10 9 9 02/30 Read block 461 len 20480 03/30 Write block 323 len 12288 data: 8 10 6 04/30 Write block 266 len 40960 data: 9 9 13 11 13 13 14 11 10 10 05/30 Write block 102 len 24576 data: 16 15 13 9 8 7 06/30 Flush 07/30 Write block 62 len 40960 data: 11 12 12 13 14 14 13 13 12 11 08/30 Write block 149 len 40960 data: 12 12 11 11 13 12 10 10 10 11 09/30 Write block 408 len 36864 data: 15 12 13 12 10 10 11 10 12 10/30 Flush 11/30 Write block 288 len 32768 data: 10 9 7 7 11 11 9 8 12/30 Read block 465 len 36864 13/30 Read block 560 len 4096 14/30 Read block 360 len 28672 15/30 Read block 529 len 4096 16/30 Flush 17/30 Write block 513 len 8192 data: 12 10 18/30 Write block 516 len 32768 data: 9 12 13 12 12 12 11 10 19/30 Flush 20/30 Flush 21/30 Write block 345 len 20480 data: 7 6 7 8 8 22/30 Read block 290 len 12288 23/30 Write block 344 len 12288 data: 7 8 7 24/30 Write block 332 len 8192 data: 9 10 25/30 Read block 245 len 28672 26/30 Write block 340 len 20480 data: 14 15 12 10 8 27/30 Write block 333 len 28672 data: 11 10 12 10 11 14 14 28/30 Write block 108 len 4096 data: 9 29/30 Read block 457 len 28672 30/30 Write block 476 len 36864 data: 7 6 6 7 9 11 11 10 11 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:28.901139594Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1688} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 122 122 122 194 194 194 F F F 3 060-079 124 124 124 197 197 197 F F F 4 080-099 121 121 121 193 193 193 F F F 5 100-119 124 124 124 196 196 196 T T T 6 120-139 123 123 123 195 195 195 F F F 7 140-159 124 124 124 197 197 197 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 123 123 123 195 195 195 F F F 10 200-219 120 120 120 191 191 191 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 120 120 120 192 192 192 F F F 13 260-279 124 124 124 196 196 196 F F F 14 280-299 124 124 124 198 198 198 F F F 15 300-319 123 123 123 195 195 195 F F F 16 320-339 124 124 124 196 196 196 T T T 17 340-359 118 118 118 186 186 186 T T T 18 360-379 122 122 122 194 194 194 F F F 19 380-399 122 122 122 194 194 194 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 122 122 122 194 194 194 F F F 22 440-459 123 123 123 195 195 195 F F F 23 460-479 123 123 123 195 195 195 T T T 24 480-499 123 123 123 195 195 195 T T T 25 500-519 124 124 124 199 199 199 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 122 122 122 194 194 194 F F F 28 560-579 122 122 122 194 194 194 F F F 29 580-599 124 124 124 196 196 196 F F F Max gen: 124, Max flush: 199 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 122 122 122 194 194 194 F F F 3 060-079 124 124 124 197 197 197 F F F 4 080-099 121 121 121 193 193 193 F F F 5 100-119 125 125 125 200 200 200 F F F 6 120-139 123 123 123 195 195 195 F F F 7 140-159 124 124 124 197 197 197 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 123 123 123 195 195 195 F F F 10 200-219 120 120 120 191 191 191 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 120 120 120 192 192 192 F F F 13 260-279 124 124 124 196 196 196 F F F 14 280-299 124 124 124 198 198 198 F F F 15 300-319 123 123 123 195 195 195 F F F 16 320-339 125 125 125 200 200 200 F F F 17 340-359 125 125 125 200 200 200 F F F 18 360-379 122 122 122 194 194 194 F F F 19 380-399 122 122 122 194 194 194 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 122 122 122 194 194 194 F F F 22 440-459 123 123 123 195 195 195 F F F 23 460-479 125 125 125 200 200 200 F F F 24 480-499 125 125 125 200 200 200 F F F 25 500-519 124 124 124 199 199 199 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 122 122 122 194 194 194 F F F 28 560-579 122 122 122 194 194 194 F F F 29 580-599 124 124 124 196 196 196 F F F Max gen: 125, Max flush: 200 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.1368113Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.180764432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.181433912Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.181475038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.181491786Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.181516353Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696} {"msg":"Crucible 2f209a37-fdd3-4dce-b781-2ed4732accd8 has session id: 706f1735-64b3-43eb-a85c-6ee6a81e5193","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182129639Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Upstairs opts: Upstairs UUID: 2f209a37-fdd3-4dce-b781-2ed4732accd8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182171804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Crucible stats registered with UUID: 2f209a37-fdd3-4dce-b781-2ed4732accd8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182196702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:43.18221745Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182651224Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182672302Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696} {"msg":"2f209a37-fdd3-4dce-b781-2ed4732accd8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182749134Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182766602Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"0","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182795549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"0","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182816207Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"1","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182842194Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"1","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182867002Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"2","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182921246Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"2","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182965641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.182993638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.183018586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"ds_connection connected from Ok(127.0.0.1:62186)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.183044103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"ds_connection connected from Ok(127.0.0.1:57197)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.18307033Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"ds_connection connected from Ok(127.0.0.1:50484)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.183095447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.183669987Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"1","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.183790074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"0","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.184192232Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"1","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.184298651Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"0","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.184832725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.184946733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.185428942Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"2","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.18592632Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"2","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[0]R flush_numbers[0..12]: [195, 194, 194, 197, 193, 200, 195, 197, 195, 195, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186424108Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[0]R generation[0..12]: [123, 122, 122, 124, 121, 125, 123, 124, 123, 123, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186448115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186464024Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[1]R flush_numbers[0..12]: [195, 194, 194, 197, 193, 200, 195, 197, 195, 195, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186479392Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[1]R generation[0..12]: [123, 122, 122, 124, 121, 125, 123, 124, 123, 123, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186494161Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186509199Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[2]R flush_numbers[0..12]: [195, 194, 194, 197, 193, 200, 195, 197, 195, 195, 191, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186523018Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[2]R generation[0..12]: [123, 122, 122, 124, 121, 125, 123, 124, 123, 123, 120, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186536396Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186549995Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186562683Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186575532Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.1865955Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186609368Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"":"downstairs","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186623017Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186635636Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"0","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186651414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"1","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186665572Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"client":"2","":"downstairs","session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186679191Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.18669138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"2f209a37-fdd3-4dce-b781-2ed4732accd8 is now active with session: 706f1735-64b3-43eb-a85c-6ee6a81e5193","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186704028Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186715777Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696,"session_id":"706f1735-64b3-43eb-a85c-6ee6a81e5193"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186727646Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696} {"msg":"Activated sub_volume 2f209a37-fdd3-4dce-b781-2ed4732accd8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:43.186738475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 387 len 20480 data: 9 9 9 10 12 02/30 Read block 215 len 32768 03/30 Write block 492 len 8192 data: 8 9 04/30 Read block 159 len 16384 05/30 Read block 519 len 20480 06/30 Write block 44 len 40960 data: 12 12 13 13 12 13 13 11 13 12 07/30 Write block 209 len 40960 data: 9 9 10 10 12 12 12 14 15 14 08/30 Read block 370 len 12288 09/30 Write block 566 len 28672 data: 10 8 9 10 6 6 8 10/30 Read block 436 len 4096 11/30 Write block 242 len 36864 data: 9 12 10 10 8 10 10 8 8 12/30 Read block 0 len 4096 13/30 Write block 516 len 8192 data: 10 13 14/30 Read block 381 len 36864 15/30 Read block 540 len 8192 16/30 Read block 267 len 36864 17/30 Read block 389 len 4096 18/30 Write block 335 len 32768 data: 13 11 12 15 15 15 16 13 19/30 Read block 155 len 28672 20/30 Write block 565 len 12288 data: 10 11 9 21/30 Read block 598 len 8192 22/30 Read block 17 len 32768 23/30 Write block 317 len 4096 data: 10 24/30 Read block 551 len 36864 25/30 Read block 211 len 36864 26/30 Flush 27/30 Write block 500 len 40960 data: 7 8 8 8 10 11 12 12 11 12 28/30 Read block 30 len 32768 29/30 Write block 282 len 24576 data: 13 14 15 14 13 11 30/30 Read block 525 len 24576 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:43.209021624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1696} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 126 126 126 201 201 201 F F F 3 060-079 124 124 124 197 197 197 F F F 4 080-099 121 121 121 193 193 193 F F F 5 100-119 125 125 125 200 200 200 F F F 6 120-139 123 123 123 195 195 195 F F F 7 140-159 124 124 124 197 197 197 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 123 123 123 195 195 195 F F F 10 200-219 126 126 126 201 201 201 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 126 126 126 201 201 201 F F F 13 260-279 124 124 124 196 196 196 F F F 14 280-299 124 124 124 198 198 198 T T T 15 300-319 126 126 126 201 201 201 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 126 126 126 201 201 201 F F F 18 360-379 122 122 122 194 194 194 F F F 19 380-399 126 126 126 201 201 201 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 122 122 122 194 194 194 F F F 22 440-459 123 123 123 195 195 195 F F F 23 460-479 125 125 125 200 200 200 F F F 24 480-499 126 126 126 201 201 201 F F F 25 500-519 126 126 126 201 201 201 T T T 26 520-539 124 124 124 199 199 199 F F F 27 540-559 122 122 122 194 194 194 F F F 28 560-579 126 126 126 201 201 201 F F F 29 580-599 124 124 124 196 196 196 F F F Max gen: 126, Max flush: 201 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 126 126 126 201 201 201 F F F 3 060-079 124 124 124 197 197 197 F F F 4 080-099 121 121 121 193 193 193 F F F 5 100-119 125 125 125 200 200 200 F F F 6 120-139 123 123 123 195 195 195 F F F 7 140-159 124 124 124 197 197 197 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 123 123 123 195 195 195 F F F 10 200-219 126 126 126 201 201 201 F F F 11 220-239 116 116 116 183 183 183 F F F 12 240-259 126 126 126 201 201 201 F F F 13 260-279 124 124 124 196 196 196 F F F 14 280-299 127 127 127 202 202 202 F F F 15 300-319 126 126 126 201 201 201 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 126 126 126 201 201 201 F F F 18 360-379 122 122 122 194 194 194 F F F 19 380-399 126 126 126 201 201 201 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 122 122 122 194 194 194 F F F 22 440-459 123 123 123 195 195 195 F F F 23 460-479 125 125 125 200 200 200 F F F 24 480-499 126 126 126 201 201 201 F F F 25 500-519 127 127 127 202 202 202 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 122 122 122 194 194 194 F F F 28 560-579 126 126 126 201 201 201 F F F 29 580-599 124 124 124 196 196 196 F F F Max gen: 127, Max flush: 202 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.432441103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.476436631Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.477067535Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.47711088Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.477128388Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.477146156Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704} {"msg":"Crucible c945a075-9517-457f-a8c4-4c2de6c141fb has session id: e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.477835424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Upstairs opts: Upstairs UUID: c945a075-9517-457f-a8c4-4c2de6c141fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.477883369Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Crucible stats registered with UUID: c945a075-9517-457f-a8c4-4c2de6c141fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.477906047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:47.477927134Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478338501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478360949Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704} {"msg":"c945a075-9517-457f-a8c4-4c2de6c141fb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478430221Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.47844728Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"0","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478478356Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"0","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478501584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478527171Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478552859Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"2","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478579696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"2","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478603303Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478626631Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478649868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37098)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478671846Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60372)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478693744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:47.478722801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:47.478753018Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:47.478783964Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.478968455Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.479377692Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"2","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.479403529Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"0","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.479941373Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"0","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.480033543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"2","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.48044501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:47.480554658Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.480106969Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35291)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.480213418Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.481567476Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.482169222Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[0]R flush_numbers[0..12]: [195, 194, 201, 197, 193, 200, 195, 197, 195, 195, 201, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.482801086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[0]R generation[0..12]: [123, 122, 126, 124, 121, 125, 123, 124, 123, 123, 126, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.482836303Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.48286211Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[1]R flush_numbers[0..12]: [195, 194, 201, 197, 193, 200, 195, 197, 195, 195, 201, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.482887697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[1]R generation[0..12]: [123, 122, 126, 124, 121, 125, 123, 124, 123, 123, 126, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.482918064Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.482943641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[2]R flush_numbers[0..12]: [195, 194, 201, 197, 193, 200, 195, 197, 195, 195, 201, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.482971038Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[2]R generation[0..12]: [123, 122, 126, 124, 121, 125, 123, 124, 123, 123, 126, 116]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.482997795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483025143Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.48304989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483075497Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483110184Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483136041Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"":"downstairs","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483154659Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483166768Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"0","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483180106Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"1","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483192865Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"client":"2","":"downstairs","session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483205624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483217563Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"c945a075-9517-457f-a8c4-4c2de6c141fb is now active with session: e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483233651Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483245609Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704,"session_id":"e62f0c63-8072-4f1d-b5f7-b9788dfbc0a5"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483257528Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704} {"msg":"Activated sub_volume c945a075-9517-457f-a8c4-4c2de6c141fb","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:24:57.483268477Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 97 len 12288 data: 11 11 15 02/30 Write block 278 len 40960 data: 7 9 9 11 14 15 16 15 14 12 03/30 Write block 423 len 40960 data: 9 7 10 10 13 14 17 20 19 18 04/30 Write block 589 len 40960 data: 14 11 11 10 11 10 10 9 5 4 05/30 Read block 361 len 4096 06/30 Read block 394 len 28672 07/30 Flush 08/30 Write block 270 len 16384 data: 14 14 15 12 09/30 Write block 365 len 28672 data: 8 9 9 8 9 9 9 10/30 Read block 138 len 8192 11/30 Read block 298 len 12288 12/30 Write block 561 len 28672 data: 19 17 15 13 11 12 10 13/30 Write block 119 len 36864 data: 9 8 10 10 11 10 9 8 7 14/30 Read block 505 len 24576 15/30 Read block 251 len 12288 16/30 Read block 261 len 8192 17/30 Read block 144 len 36864 18/30 Read block 188 len 36864 19/30 Write block 350 len 20480 data: 7 7 6 7 10 20/30 Read block 31 len 16384 21/30 Read block 436 len 36864 22/30 Read block 180 len 20480 23/30 Read block 221 len 36864 24/30 Read block 446 len 40960 25/30 Read block 513 len 20480 26/30 Write block 217 len 20480 data: 16 15 17 18 16 27/30 Read block 28 len 12288 28/30 Write block 214 len 16384 data: 13 13 15 17 29/30 Write block 431 len 4096 data: 20 30/30 Read block 87 len 40960 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 4 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 2 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:24:57.504623204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1704} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 126 126 126 201 201 201 F F F 3 060-079 124 124 124 197 197 197 F F F 4 080-099 128 128 128 203 203 203 F F F 5 100-119 125 125 125 200 200 200 T T T 6 120-139 123 123 123 195 195 195 T T T 7 140-159 124 124 124 197 197 197 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 123 123 123 195 195 195 F F F 10 200-219 126 126 126 201 201 201 T T T 11 220-239 116 116 116 183 183 183 T T T 12 240-259 126 126 126 201 201 201 F F F 13 260-279 128 128 128 203 203 203 T T T 14 280-299 128 128 128 203 203 203 F F F 15 300-319 126 126 126 201 201 201 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 126 126 126 201 201 201 T T T 18 360-379 122 122 122 194 194 194 T T T 19 380-399 126 126 126 201 201 201 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 128 128 128 203 203 203 T T T 22 440-459 123 123 123 195 195 195 F F F 23 460-479 125 125 125 200 200 200 F F F 24 480-499 126 126 126 201 201 201 F F F 25 500-519 127 127 127 202 202 202 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 122 122 122 194 194 194 F F F 28 560-579 126 126 126 201 201 201 T T T 29 580-599 128 128 128 203 203 203 F F F Max gen: 128, Max flush: 203 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 126 126 126 201 201 201 F F F 3 060-079 124 124 124 197 197 197 F F F 4 080-099 128 128 128 203 203 203 F F F 5 100-119 129 129 129 204 204 204 F F F 6 120-139 129 129 129 204 204 204 F F F 7 140-159 124 124 124 197 197 197 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 123 123 123 195 195 195 F F F 10 200-219 129 129 129 204 204 204 F F F 11 220-239 129 129 129 204 204 204 F F F 12 240-259 126 126 126 201 201 201 F F F 13 260-279 129 129 129 204 204 204 F F F 14 280-299 128 128 128 203 203 203 F F F 15 300-319 126 126 126 201 201 201 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 129 129 129 204 204 204 F F F 19 380-399 126 126 126 201 201 201 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 129 129 129 204 204 204 F F F 22 440-459 123 123 123 195 195 195 F F F 23 460-479 125 125 125 200 200 200 F F F 24 480-499 126 126 126 201 201 201 F F F 25 500-519 127 127 127 202 202 202 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 122 122 122 194 194 194 F F F 28 560-579 129 129 129 204 204 204 F F F 29 580-599 128 128 128 203 203 203 F F F Max gen: 129, Max flush: 204 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.765796378Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.812258506Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.812905838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.812939755Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.812953563Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.812968012Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712} {"msg":"Crucible c5e4e094-f8e3-4d90-b1d1-75873b361486 has session id: 2b923a3e-528c-4ab7-9d2e-642fc9c275ca","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.813668218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Upstairs opts: Upstairs UUID: c5e4e094-f8e3-4d90-b1d1-75873b361486, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.813707404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Crucible stats registered with UUID: c5e4e094-f8e3-4d90-b1d1-75873b361486","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.813731222Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:11.81374787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.81421789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814245868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712} {"msg":"c5e4e094-f8e3-4d90-b1d1-75873b361486 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.81431551Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814332898Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"0","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814364445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"0","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814378814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"1","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814393782Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"1","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.81441708Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"2","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814443767Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"2","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814466024Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814484622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.81450943Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:47571)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814599201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:58152)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814636517Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:45817)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.814661294Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.815297207Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"2","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.815327414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"0","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.815978896Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"0","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.816120871Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"2","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.816438837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"1","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.816644675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.816743055Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817009857Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"1","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[0]R flush_numbers[0..12]: [195, 194, 201, 197, 203, 204, 204, 197, 195, 195, 204, 204]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817508155Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[0]R generation[0..12]: [123, 122, 126, 124, 128, 129, 129, 124, 123, 123, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817528823Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817543591Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[1]R flush_numbers[0..12]: [195, 194, 201, 197, 203, 204, 204, 197, 195, 195, 204, 204]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.8175582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[1]R generation[0..12]: [123, 122, 126, 124, 128, 129, 129, 124, 123, 123, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817581417Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817615204Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[2]R flush_numbers[0..12]: [195, 194, 201, 197, 203, 204, 204, 197, 195, 195, 204, 204]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817643831Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[2]R generation[0..12]: [123, 122, 126, 124, 128, 129, 129, 124, 123, 123, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817675207Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817708084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.8177416Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817766138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 205","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817803114Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817834371Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"":"downstairs","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817858658Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817875066Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"0","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817902934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"1","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.81793368Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"client":"2","":"downstairs","session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817958897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.817979815Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"c5e4e094-f8e3-4d90-b1d1-75873b361486 is now active with session: 2b923a3e-528c-4ab7-9d2e-642fc9c275ca","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.818007432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.81803452Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712,"session_id":"2b923a3e-528c-4ab7-9d2e-642fc9c275ca"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.818062177Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712} {"msg":"Activated sub_volume c5e4e094-f8e3-4d90-b1d1-75873b361486","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:11.818077545Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 385 len 4096 data: 9 02/30 Flush 03/30 Write block 542 len 12288 data: 10 13 12 04/30 Read block 374 len 4096 05/30 Read block 161 len 4096 06/30 Write block 228 len 16384 data: 6 6 5 8 07/30 Read block 556 len 28672 08/30 Read block 107 len 16384 09/30 Write block 482 len 36864 data: 12 11 12 13 14 13 12 9 7 10/30 Write block 55 len 16384 data: 13 13 11 10 11/30 Read block 363 len 20480 12/30 Read block 167 len 36864 13/30 Read block 48 len 20480 14/30 Flush 15/30 Write block 58 len 16384 data: 11 10 9 8 16/30 Read block 598 len 8192 17/30 Flush 18/30 Write block 589 len 12288 data: 15 12 12 19/30 Write block 580 len 32768 data: 9 13 12 11 14 11 14 14 20/30 Read block 215 len 40960 21/30 Write block 276 len 16384 data: 7 8 8 10 22/30 Write block 452 len 32768 data: 12 11 14 14 16 14 14 16 23/30 Write block 509 len 36864 data: 13 14 16 13 13 11 11 11 14 24/30 Write block 135 len 28672 data: 10 11 12 12 13 12 13 25/30 Write block 196 len 24576 data: 12 10 9 9 12 11 26/30 Read block 189 len 36864 27/30 Write block 85 len 12288 data: 8 8 8 28/30 Read block 280 len 8192 29/30 Read block 487 len 4096 30/30 Read block 379 len 8192 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 12 13 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:11.837308715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 130 130 130 207 207 207 F F F 3 060-079 130 130 130 207 207 207 F F F 4 080-099 128 128 128 203 203 203 T T T 5 100-119 129 129 129 204 204 204 F F F 6 120-139 129 129 129 204 204 204 T T T 7 140-159 124 124 124 197 197 197 T T T 8 160-179 123 123 123 195 195 195 F F F 9 180-199 123 123 123 195 195 195 T T T 10 200-219 129 129 129 204 204 204 T T T 11 220-239 130 130 130 206 206 206 F F F 12 240-259 126 126 126 201 201 201 F F F 13 260-279 129 129 129 204 204 204 T T T 14 280-299 128 128 128 203 203 203 F F F 15 300-319 126 126 126 201 201 201 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 129 129 129 204 204 204 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 129 129 129 204 204 204 F F F 22 440-459 123 123 123 195 195 195 T T T 23 460-479 125 125 125 200 200 200 F F F 24 480-499 130 130 130 206 206 206 F F F 25 500-519 127 127 127 202 202 202 T T T 26 520-539 124 124 124 199 199 199 F F F 27 540-559 130 130 130 206 206 206 F F F 28 560-579 129 129 129 204 204 204 F F F 29 580-599 128 128 128 203 203 203 T T T Max gen: 130, Max flush: 207 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 122 122 122 194 194 194 F F F 2 040-059 130 130 130 207 207 207 F F F 3 060-079 130 130 130 207 207 207 F F F 4 080-099 131 131 131 208 208 208 F F F 5 100-119 129 129 129 204 204 204 F F F 6 120-139 131 131 131 208 208 208 F F F 7 140-159 131 131 131 208 208 208 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 131 131 131 208 208 208 F F F 10 200-219 131 131 131 208 208 208 F F F 11 220-239 130 130 130 206 206 206 F F F 12 240-259 126 126 126 201 201 201 F F F 13 260-279 131 131 131 208 208 208 F F F 14 280-299 128 128 128 203 203 203 F F F 15 300-319 126 126 126 201 201 201 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 129 129 129 204 204 204 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 129 129 129 204 204 204 F F F 22 440-459 131 131 131 208 208 208 F F F 23 460-479 125 125 125 200 200 200 F F F 24 480-499 130 130 130 206 206 206 F F F 25 500-519 131 131 131 208 208 208 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 130 130 130 206 206 206 F F F 28 560-579 129 129 129 204 204 204 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 131, Max flush: 208 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.093548608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.138139614Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.13875151Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.138785606Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.138800034Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.138814363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720} {"msg":"Crucible 737687c0-05d9-4e8e-bce7-01e865371110 has session id: d49135c3-e954-4c4d-ad23-49d6221640b0","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.139368585Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Upstairs opts: Upstairs UUID: 737687c0-05d9-4e8e-bce7-01e865371110, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.139417989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Crucible stats registered with UUID: 737687c0-05d9-4e8e-bce7-01e865371110","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.139441377Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:16.139462105Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.14007705Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140102688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720} {"msg":"737687c0-05d9-4e8e-bce7-01e865371110 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.14017322Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140189828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"0","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140217616Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"0","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140231474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"1","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140245673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"1","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140260971Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"2","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140285798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"2","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140310286Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140336903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.14036495Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:61835)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140399366Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:55434)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.140428933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:48236)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.14065412Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.141100533Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"0","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.141267665Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"1","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.141335478Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"2","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.141688931Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"0","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.141807858Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"1","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142207936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"2","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142399686Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142424624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[0]R flush_numbers[0..12]: [195, 194, 207, 207, 208, 204, 208, 208, 195, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142713533Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[0]R generation[0..12]: [123, 122, 130, 130, 131, 129, 131, 131, 123, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142733341Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.14274782Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[1]R flush_numbers[0..12]: [195, 194, 207, 207, 208, 204, 208, 208, 195, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142766008Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[1]R generation[0..12]: [123, 122, 130, 130, 131, 129, 131, 131, 123, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142779846Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142793435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[2]R flush_numbers[0..12]: [195, 194, 207, 207, 208, 204, 208, 208, 195, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142806784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[2]R generation[0..12]: [123, 122, 130, 130, 131, 129, 131, 131, 123, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142820132Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142833671Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.14284612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142858988Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142871717Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:25:16.142884305Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"":"downstairs","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142900734Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142912542Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"0","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142925701Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"1","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.14293849Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"client":"2","":"downstairs","session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142951438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142965457Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"737687c0-05d9-4e8e-bce7-01e865371110 is now active with session: d49135c3-e954-4c4d-ad23-49d6221640b0","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142978086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.142989524Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720,"session_id":"d49135c3-e954-4c4d-ad23-49d6221640b0"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.143001263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720} {"msg":"Activated sub_volume 737687c0-05d9-4e8e-bce7-01e865371110","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:16.143012262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 421 len 28672 data: 8 10 10 8 11 11 14 02/30 Read block 187 len 20480 03/30 Read block 69 len 20480 04/30 Read block 279 len 20480 05/30 Write block 470 len 28672 data: 19 19 15 15 11 7 8 06/30 Read block 175 len 20480 07/30 Read block 554 len 12288 08/30 Read block 487 len 36864 09/30 Read block 545 len 24576 10/30 Read block 224 len 40960 11/30 Write block 317 len 4096 data: 11 12/30 Read block 82 len 28672 13/30 Flush 14/30 Write block 74 len 8192 data: 8 7 15/30 Read block 354 len 4096 16/30 Write block 29 len 20480 data: 10 13 13 13 12 17/30 Write block 573 len 16384 data: 9 8 9 9 18/30 Read block 86 len 28672 19/30 Read block 165 len 24576 20/30 Read block 51 len 36864 21/30 Read block 293 len 20480 22/30 Read block 25 len 16384 23/30 Read block 546 len 40960 24/30 Flush 25/30 Write block 252 len 16384 data: 11 9 10 11 26/30 Write block 257 len 24576 data: 5 5 4 5 7 7 27/30 Read block 455 len 24576 28/30 Read block 96 len 4096 29/30 Read block 446 len 24576 30/30 Read block 48 len 40960 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:16.163718757Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1720} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 132 132 132 210 210 210 F F F 2 040-059 130 130 130 207 207 207 F F F 3 060-079 132 132 132 210 210 210 F F F 4 080-099 131 131 131 208 208 208 F F F 5 100-119 129 129 129 204 204 204 F F F 6 120-139 131 131 131 208 208 208 F F F 7 140-159 131 131 131 208 208 208 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 131 131 131 208 208 208 F F F 10 200-219 131 131 131 208 208 208 F F F 11 220-239 130 130 130 206 206 206 F F F 12 240-259 126 126 126 201 201 201 T T T 13 260-279 131 131 131 208 208 208 T T T 14 280-299 128 128 128 203 203 203 F F F 15 300-319 132 132 132 209 209 209 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 129 129 129 204 204 204 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 132 132 132 209 209 209 F F F 22 440-459 131 131 131 208 208 208 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 130 130 130 206 206 206 F F F 25 500-519 131 131 131 208 208 208 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 130 130 130 206 206 206 F F F 28 560-579 132 132 132 210 210 210 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 132, Max flush: 210 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 F F F 1 020-039 132 132 132 210 210 210 F F F 2 040-059 130 130 130 207 207 207 F F F 3 060-079 132 132 132 210 210 210 F F F 4 080-099 131 131 131 208 208 208 F F F 5 100-119 129 129 129 204 204 204 F F F 6 120-139 131 131 131 208 208 208 F F F 7 140-159 131 131 131 208 208 208 F F F 8 160-179 123 123 123 195 195 195 F F F 9 180-199 131 131 131 208 208 208 F F F 10 200-219 131 131 131 208 208 208 F F F 11 220-239 130 130 130 206 206 206 F F F 12 240-259 133 133 133 211 211 211 F F F 13 260-279 133 133 133 211 211 211 F F F 14 280-299 128 128 128 203 203 203 F F F 15 300-319 132 132 132 209 209 209 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 129 129 129 204 204 204 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 132 132 132 209 209 209 F F F 22 440-459 131 131 131 208 208 208 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 130 130 130 206 206 206 F F F 25 500-519 131 131 131 208 208 208 F F F 26 520-539 124 124 124 199 199 199 F F F 27 540-559 130 130 130 206 206 206 F F F 28 560-579 132 132 132 210 210 210 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 133, Max flush: 211 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.381948812Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.426491123Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.427129525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.427169021Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.42718439Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.427198828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728} {"msg":"Crucible 6d024708-c87a-453b-84c0-effc5eeb6529 has session id: 2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.42784936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Upstairs opts: Upstairs UUID: 6d024708-c87a-453b-84c0-effc5eeb6529, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.427883676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Crucible stats registered with UUID: 6d024708-c87a-453b-84c0-effc5eeb6529","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.427899105Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:20.427911753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428474754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428508141Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728} {"msg":"6d024708-c87a-453b-84c0-effc5eeb6529 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428574904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.42861023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"0","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428654275Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"0","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428681142Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"1","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.42871053Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"1","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428742126Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"2","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428775753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"2","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.42880124Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428827997Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428851665Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36878)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428875032Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:52781)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.42889892Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58483)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.428967712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.429583248Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"0","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.429628693Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"1","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.429672918Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"2","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.430273095Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"0","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.430313261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"1","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.430344718Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"2","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.430954074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.43098814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[0]R flush_numbers[0..12]: [195, 210, 207, 210, 208, 204, 208, 208, 195, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431044674Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[0]R generation[0..12]: [123, 132, 130, 132, 131, 129, 131, 131, 123, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431072541Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.43108782Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[1]R flush_numbers[0..12]: [195, 210, 207, 210, 208, 204, 208, 208, 195, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431108487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[1]R generation[0..12]: [123, 132, 130, 132, 131, 129, 131, 131, 123, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431134915Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431161542Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[2]R flush_numbers[0..12]: [195, 210, 207, 210, 208, 204, 208, 208, 195, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.43118375Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[2]R generation[0..12]: [123, 132, 130, 132, 131, 129, 131, 131, 123, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431198848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431216686Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431239904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431265361Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431287939Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431304877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"":"downstairs","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431326645Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431355182Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"0","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431380359Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"1","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431396747Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"client":"2","":"downstairs","session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431410016Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431430154Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"6d024708-c87a-453b-84c0-effc5eeb6529 is now active with session: 2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431453921Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431474649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728,"session_id":"2c0cab82-bc0e-4762-86f7-b7d13a9cd9fd"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431498597Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728} {"msg":"Activated sub_volume 6d024708-c87a-453b-84c0-effc5eeb6529","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:20.431517964Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 536 len 12288 data: 8 9 9 02/30 Write block 529 len 20480 data: 10 9 8 9 10 03/30 Read block 307 len 12288 04/30 Write block 436 len 28672 data: 11 9 9 7 8 7 7 05/30 Write block 561 len 20480 data: 20 18 16 14 12 06/30 Write block 265 len 28672 data: 8 10 10 14 12 15 15 07/30 Write block 285 len 12288 data: 16 15 13 08/30 Write block 257 len 28672 data: 6 6 5 6 8 8 7 09/30 Write block 100 len 4096 data: 16 10/30 Flush 11/30 Write block 113 len 12288 data: 8 9 7 12/30 Flush 13/30 Write block 494 len 28672 data: 10 7 6 6 5 5 8 14/30 Read block 549 len 40960 15/30 Read block 248 len 20480 16/30 Flush 17/30 Write block 420 len 36864 data: 13 9 11 11 9 12 12 15 15 18/30 Read block 31 len 16384 19/30 Read block 166 len 24576 20/30 Write block 106 len 8192 data: 9 8 21/30 Write block 84 len 8192 data: 8 9 22/30 Flush 23/30 Write block 550 len 16384 data: 11 11 14 14 24/30 Write block 27 len 16384 data: 6 7 11 14 25/30 Write block 163 len 4096 data: 15 26/30 Write block 249 len 40960 data: 9 9 9 12 10 11 12 8 7 7 27/30 Read block 320 len 28672 28/30 Read block 164 len 40960 29/30 Read block 589 len 8192 30/30 Write block 19 len 24576 data: 8 8 8 8 8 7 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:20.449226244Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1728} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 195 195 195 T T T 1 020-039 132 132 132 210 210 210 T T T 2 040-059 130 130 130 207 207 207 F F F 3 060-079 132 132 132 210 210 210 F F F 4 080-099 134 134 134 215 215 215 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 131 131 131 208 208 208 F F F 7 140-159 131 131 131 208 208 208 F F F 8 160-179 123 123 123 195 195 195 T T T 9 180-199 131 131 131 208 208 208 F F F 10 200-219 131 131 131 208 208 208 F F F 11 220-239 130 130 130 206 206 206 F F F 12 240-259 134 134 134 212 212 212 T T T 13 260-279 134 134 134 212 212 212 F F F 14 280-299 134 134 134 212 212 212 F F F 15 300-319 132 132 132 209 209 209 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 129 129 129 204 204 204 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 134 134 134 215 215 215 F F F 22 440-459 134 134 134 212 212 212 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 134 134 134 214 214 214 F F F 25 500-519 134 134 134 214 214 214 F F F 26 520-539 134 134 134 212 212 212 F F F 27 540-559 130 130 130 206 206 206 T T T 28 560-579 134 134 134 212 212 212 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 134, Max flush: 215 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 216 216 216 F F F 1 020-039 135 135 135 216 216 216 F F F 2 040-059 130 130 130 207 207 207 F F F 3 060-079 132 132 132 210 210 210 F F F 4 080-099 134 134 134 215 215 215 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 131 131 131 208 208 208 F F F 7 140-159 131 131 131 208 208 208 F F F 8 160-179 135 135 135 216 216 216 F F F 9 180-199 131 131 131 208 208 208 F F F 10 200-219 131 131 131 208 208 208 F F F 11 220-239 130 130 130 206 206 206 F F F 12 240-259 135 135 135 216 216 216 F F F 13 260-279 134 134 134 212 212 212 F F F 14 280-299 134 134 134 212 212 212 F F F 15 300-319 132 132 132 209 209 209 F F F 16 320-339 126 126 126 201 201 201 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 129 129 129 204 204 204 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 124 124 124 197 197 197 F F F 21 420-439 134 134 134 215 215 215 F F F 22 440-459 134 134 134 212 212 212 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 134 134 134 214 214 214 F F F 25 500-519 134 134 134 214 214 214 F F F 26 520-539 134 134 134 212 212 212 F F F 27 540-559 135 135 135 216 216 216 F F F 28 560-579 134 134 134 212 212 212 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 135, Max flush: 216 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.683981553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:24.726175341Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.726535373Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.758569578Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.759189173Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.759222829Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.759236668Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.759250566Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736} {"msg":"Crucible d89bc4c1-3c09-4b7b-8642-86d416aa76cb has session id: 363861f5-f7dc-4e63-9d8a-df776843d57d","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.759815497Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Upstairs opts: Upstairs UUID: d89bc4c1-3c09-4b7b-8642-86d416aa76cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.759854033Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Crucible stats registered with UUID: d89bc4c1-3c09-4b7b-8642-86d416aa76cb","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.75987929Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:24.759893899Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760310335Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760333342Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736} {"msg":"d89bc4c1-3c09-4b7b-8642-86d416aa76cb active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760405065Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760422743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"0","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.76044958Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"0","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760464148Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"1","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760479037Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"1","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760492026Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"2","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760505754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"2","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760518373Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760531432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.7605444Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38503)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760650549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"ds_connection connected from Ok(127.0.0.1:64790)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760689105Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"ds_connection connected from Ok(127.0.0.1:47654)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.760724121Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.761370743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"1","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.76140232Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"2","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.761559073Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"0","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.761924965Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"1","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.761967221Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"2","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762307445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"0","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762555649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.76264213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[0]R flush_numbers[0..12]: [216, 216, 207, 210, 215, 215, 208, 208, 216, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762873885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[0]R generation[0..12]: [135, 135, 130, 132, 134, 134, 131, 131, 135, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762894523Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762909062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[1]R flush_numbers[0..12]: [216, 216, 207, 210, 215, 215, 208, 208, 216, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762936279Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[1]R generation[0..12]: [135, 135, 130, 132, 134, 134, 131, 131, 135, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762950097Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762968875Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[2]R flush_numbers[0..12]: [216, 216, 207, 210, 215, 215, 208, 208, 216, 208, 208, 206]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762982784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[2]R generation[0..12]: [135, 135, 130, 132, 134, 134, 131, 131, 135, 131, 131, 130]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.762996282Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763009961Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.76302261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763036978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763049617Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763062096Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763081353Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763093072Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"0","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763107011Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"1","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.76311988Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"client":"2","":"downstairs","session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763132728Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763144607Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"d89bc4c1-3c09-4b7b-8642-86d416aa76cb is now active with session: 363861f5-f7dc-4e63-9d8a-df776843d57d","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763157146Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763170984Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736,"session_id":"363861f5-f7dc-4e63-9d8a-df776843d57d"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763183103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736} {"msg":"Activated sub_volume d89bc4c1-3c09-4b7b-8642-86d416aa76cb","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:24.763194162Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 491 len 36864 data: 6 9 10 11 8 7 7 6 6 02/30 Flush 03/30 Write block 499 len 12288 data: 7 9 9 04/30 Flush 05/30 Write block 332 len 32768 data: 10 12 11 14 12 13 16 16 06/30 Read block 505 len 36864 07/30 Read block 364 len 28672 08/30 Write block 140 len 16384 data: 13 14 14 15 09/30 Write block 232 len 4096 data: 10 10/30 Flush 11/30 Write block 77 len 20480 data: 11 9 7 8 8 12/30 Write block 339 len 4096 data: 17 13/30 Flush 14/30 Write block 135 len 32768 data: 11 12 13 13 14 14 15 15 15/30 Write block 411 len 4096 data: 13 16/30 Read block 399 len 36864 17/30 Write block 12 len 12288 data: 11 9 9 18/30 Read block 72 len 12288 19/30 Write block 246 len 28672 data: 9 11 11 10 10 10 13 20/30 Write block 173 len 32768 data: 8 12 11 7 8 6 7 9 21/30 Flush 22/30 Write block 79 len 16384 data: 8 9 9 9 23/30 Write block 412 len 16384 data: 11 11 12 11 24/30 Read block 143 len 32768 25/30 Write block 16 len 36864 data: 10 11 9 9 9 9 9 9 8 26/30 Write block 330 len 36864 data: 11 10 11 13 12 15 13 14 17 27/30 Read block 115 len 28672 28/30 Read block 563 len 40960 29/30 Write block 360 len 8192 data: 7 6 30/30 Read block 547 len 40960 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 9 8 8 25 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:24.783650072Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1736} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 221 221 221 T T T 1 020-039 135 135 135 216 216 216 T T T 2 040-059 130 130 130 207 207 207 F F F 3 060-079 136 136 136 220 220 220 T T T 4 080-099 136 136 136 220 220 220 T T T 5 100-119 134 134 134 215 215 215 F F F 6 120-139 136 136 136 221 221 221 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 136 136 136 221 221 221 F F F 9 180-199 136 136 136 221 221 221 F F F 10 200-219 131 131 131 208 208 208 F F F 11 220-239 136 136 136 219 219 219 F F F 12 240-259 136 136 136 221 221 221 F F F 13 260-279 134 134 134 212 212 212 F F F 14 280-299 134 134 134 212 212 212 F F F 15 300-319 132 132 132 209 209 209 F F F 16 320-339 136 136 136 220 220 220 T T T 17 340-359 129 129 129 204 204 204 F F F 18 360-379 129 129 129 204 204 204 T T T 19 380-399 130 130 130 205 205 205 F F F 20 400-419 136 136 136 221 221 221 T T T 21 420-439 134 134 134 215 215 215 F F F 22 440-459 134 134 134 212 212 212 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 136 136 136 218 218 218 F F F 25 500-519 136 136 136 218 218 218 F F F 26 520-539 134 134 134 212 212 212 F F F 27 540-559 135 135 135 216 216 216 F F F 28 560-579 134 134 134 212 212 212 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 136, Max flush: 221 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 222 222 222 F F F 1 020-039 137 137 137 222 222 222 F F F 2 040-059 130 130 130 207 207 207 F F F 3 060-079 137 137 137 222 222 222 F F F 4 080-099 137 137 137 222 222 222 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 136 136 136 221 221 221 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 136 136 136 221 221 221 F F F 9 180-199 136 136 136 221 221 221 F F F 10 200-219 131 131 131 208 208 208 F F F 11 220-239 136 136 136 219 219 219 F F F 12 240-259 136 136 136 221 221 221 F F F 13 260-279 134 134 134 212 212 212 F F F 14 280-299 134 134 134 212 212 212 F F F 15 300-319 132 132 132 209 209 209 F F F 16 320-339 137 137 137 222 222 222 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 137 137 137 222 222 222 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 137 137 137 222 222 222 F F F 21 420-439 134 134 134 215 215 215 F F F 22 440-459 134 134 134 212 212 212 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 136 136 136 218 218 218 F F F 25 500-519 136 136 136 218 218 218 F F F 26 520-539 134 134 134 212 212 212 F F F 27 540-559 135 135 135 216 216 216 F F F 28 560-579 134 134 134 212 212 212 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 137, Max flush: 222 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.063216384Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.107474395Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.108245064Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.10827874Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.108298878Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.108313786Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744} {"msg":"Crucible c97b0d7f-ed9a-46f9-9c04-a85e45644763 has session id: 2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.108876467Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Upstairs opts: Upstairs UUID: c97b0d7f-ed9a-46f9-9c04-a85e45644763, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.10894995Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Crucible stats registered with UUID: c97b0d7f-ed9a-46f9-9c04-a85e45644763","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.108978387Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:29.108999554Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.10942Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109446727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744} {"msg":"c97b0d7f-ed9a-46f9-9c04-a85e45644763 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109509271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109527869Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"0","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109558726Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"0","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109575754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"1","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109589922Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"1","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.1096093Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"2","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109624749Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"2","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109638007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109658735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109682393Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62727)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109822738Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:65086)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.109870513Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:38167)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.10990227Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.11056706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"0","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.110597857Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"2","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.111223781Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"0","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.111251928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"2","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.111506421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"1","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.111853914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112046614Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112169301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"1","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[0]R flush_numbers[0..12]: [222, 222, 207, 222, 222, 215, 221, 221, 221, 221, 208, 219]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112688717Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[0]R generation[0..12]: [137, 137, 130, 137, 137, 134, 136, 136, 136, 136, 131, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112707985Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112722013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[1]R flush_numbers[0..12]: [222, 222, 207, 222, 222, 215, 221, 221, 221, 221, 208, 219]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112735672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[1]R generation[0..12]: [137, 137, 130, 137, 137, 134, 136, 136, 136, 136, 131, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.1127495Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112763669Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[2]R flush_numbers[0..12]: [222, 222, 207, 222, 222, 215, 221, 221, 221, 221, 208, 219]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112777088Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[2]R generation[0..12]: [137, 137, 130, 137, 137, 134, 136, 136, 136, 136, 131, 136]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112790356Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112803805Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112816384Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112828952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112841451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112853829Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"":"downstairs","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112875917Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112888346Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"0","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112901264Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"1","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112913753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"client":"2","":"downstairs","session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112927162Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.11293902Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"c97b0d7f-ed9a-46f9-9c04-a85e45644763 is now active with session: 2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112951449Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112963018Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744,"session_id":"2f5fca6f-fb73-4d88-abf3-85dc34f8bcf2"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112974897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744} {"msg":"Activated sub_volume c97b0d7f-ed9a-46f9-9c04-a85e45644763","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:29.112985656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 84 len 36864 data: 9 10 9 9 9 8 5 6 6 02/30 Read block 152 len 4096 03/30 Write block 14 len 32768 data: 10 11 11 12 10 10 10 10 04/30 Read block 44 len 4096 05/30 Write block 538 len 36864 data: 10 8 6 8 11 14 13 11 12 06/30 Write block 400 len 16384 data: 14 13 15 16 07/30 Write block 213 len 36864 data: 13 14 14 16 18 16 18 19 17 08/30 Write block 513 len 40960 data: 14 12 12 12 15 14 13 13 13 12 09/30 Write block 197 len 4096 data: 11 10/30 Write block 42 len 16384 data: 12 12 13 13 11/30 Write block 266 len 36864 data: 11 11 15 13 16 16 16 13 11 12/30 Flush 13/30 Write block 492 len 28672 data: 10 11 12 9 8 8 7 14/30 Read block 323 len 4096 15/30 Write block 286 len 24576 data: 16 14 11 10 8 8 16/30 Read block 76 len 36864 17/30 Write block 319 len 8192 data: 11 9 18/30 Read block 297 len 16384 19/30 Read block 169 len 4096 20/30 Read block 49 len 12288 21/30 Write block 69 len 28672 data: 14 13 12 9 9 9 8 22/30 Write block 440 len 40960 data: 9 8 8 8 9 8 11 11 10 12 23/30 Read block 358 len 32768 24/30 Flush 25/30 Flush 26/30 Write block 323 len 24576 data: 9 11 7 7 8 8 27/30 Read block 163 len 40960 28/30 Read block 72 len 36864 29/30 Read block 48 len 32768 30/30 Write block 504 len 36864 data: 11 12 13 13 12 14 15 17 14 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:29.13388676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1744} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 223 223 223 F F F 1 020-039 138 138 138 223 223 223 F F F 2 040-059 138 138 138 223 223 223 F F F 3 060-079 138 138 138 224 224 224 F F F 4 080-099 138 138 138 223 223 223 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 136 136 136 221 221 221 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 136 136 136 221 221 221 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 138 138 138 223 223 223 F F F 11 220-239 138 138 138 223 223 223 F F F 12 240-259 136 136 136 221 221 221 F F F 13 260-279 138 138 138 223 223 223 F F F 14 280-299 138 138 138 224 224 224 F F F 15 300-319 138 138 138 224 224 224 F F F 16 320-339 138 138 138 224 224 224 T T T 17 340-359 129 129 129 204 204 204 F F F 18 360-379 137 137 137 222 222 222 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 138 138 138 223 223 223 F F F 21 420-439 134 134 134 215 215 215 F F F 22 440-459 138 138 138 224 224 224 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 138 138 138 224 224 224 F F F 25 500-519 138 138 138 223 223 223 T T T 26 520-539 138 138 138 223 223 223 F F F 27 540-559 138 138 138 223 223 223 F F F 28 560-579 134 134 134 212 212 212 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 138, Max flush: 224 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 223 223 223 F F F 1 020-039 138 138 138 223 223 223 F F F 2 040-059 138 138 138 223 223 223 F F F 3 060-079 138 138 138 224 224 224 F F F 4 080-099 138 138 138 223 223 223 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 136 136 136 221 221 221 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 136 136 136 221 221 221 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 138 138 138 223 223 223 F F F 11 220-239 138 138 138 223 223 223 F F F 12 240-259 136 136 136 221 221 221 F F F 13 260-279 138 138 138 223 223 223 F F F 14 280-299 138 138 138 224 224 224 F F F 15 300-319 138 138 138 224 224 224 F F F 16 320-339 139 139 139 225 225 225 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 137 137 137 222 222 222 F F F 19 380-399 130 130 130 205 205 205 F F F 20 400-419 138 138 138 223 223 223 F F F 21 420-439 134 134 134 215 215 215 F F F 22 440-459 138 138 138 224 224 224 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 138 138 138 224 224 224 F F F 25 500-519 139 139 139 225 225 225 F F F 26 520-539 138 138 138 223 223 223 F F F 27 540-559 138 138 138 223 223 223 F F F 28 560-579 134 134 134 212 212 212 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 139, Max flush: 225 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.361919158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:43.404193507Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.404585686Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.435331236Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.435913895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.435948591Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.435964239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.435981137Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752} {"msg":"Crucible 4dfa67ac-4fb8-42eb-aac7-292958dac043 has session id: ca17ea5d-4aef-433f-bb50-4a0bce07a4d2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.436632459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Upstairs opts: Upstairs UUID: 4dfa67ac-4fb8-42eb-aac7-292958dac043, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.436671375Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Crucible stats registered with UUID: 4dfa67ac-4fb8-42eb-aac7-292958dac043","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.436694992Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:43.43671933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437154394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437179751Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752} {"msg":"4dfa67ac-4fb8-42eb-aac7-292958dac043 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437250754Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437268182Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"0","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437293999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"0","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437308018Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"1","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437321947Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"1","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437334685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"2","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437348254Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"2","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437360763Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437374081Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437389839Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:33988)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437496958Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:46603)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437537544Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49344)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.437676929Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.438220512Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"2","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.438261108Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"1","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.43852099Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"0","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.438865164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"1","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.438892662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"2","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439158914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"0","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439457572Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.43948138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[0]R flush_numbers[0..12]: [223, 223, 223, 224, 223, 215, 221, 221, 221, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439793077Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[0]R generation[0..12]: [138, 138, 138, 138, 138, 134, 136, 136, 136, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439828443Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.43985553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[1]R flush_numbers[0..12]: [223, 223, 223, 224, 223, 215, 221, 221, 221, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439881358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[1]R generation[0..12]: [138, 138, 138, 138, 138, 134, 136, 136, 136, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439905425Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439934762Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[2]R flush_numbers[0..12]: [223, 223, 223, 224, 223, 215, 221, 221, 221, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439963069Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[2]R generation[0..12]: [138, 138, 138, 138, 138, 134, 136, 136, 136, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.439990426Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440018453Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440054469Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440079867Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440105294Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440130772Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"":"downstairs","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440156219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440179846Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"0","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440205264Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"1","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440231001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"client":"2","":"downstairs","session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:25:43.440257568Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440282336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"4dfa67ac-4fb8-42eb-aac7-292958dac043 is now active with session: ca17ea5d-4aef-433f-bb50-4a0bce07a4d2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440299164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440314292Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752,"session_id":"ca17ea5d-4aef-433f-bb50-4a0bce07a4d2"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.440326541Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752} {"msg":"Activated sub_volume 4dfa67ac-4fb8-42eb-aac7-292958dac043","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:43.44033777Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 79 len 16384 data: 9 10 10 10 02/30 Read block 382 len 20480 03/30 Read block 158 len 12288 04/30 Write block 325 len 4096 data: 8 05/30 Write block 396 len 8192 data: 15 13 06/30 Read block 312 len 40960 07/30 Write block 261 len 36864 data: 9 9 8 7 9 12 12 16 14 08/30 Read block 119 len 36864 09/30 Read block 154 len 32768 10/30 Write block 303 len 16384 data: 9 8 6 7 11/30 Read block 100 len 4096 12/30 Write block 515 len 28672 data: 13 13 16 15 14 14 14 13/30 Read block 143 len 28672 14/30 Read block 474 len 24576 15/30 Read block 336 len 12288 16/30 Read block 33 len 4096 17/30 Flush 18/30 Write block 278 len 12288 data: 9 11 10 19/30 Read block 128 len 36864 20/30 Read block 240 len 8192 21/30 Write block 313 len 16384 data: 11 13 13 10 22/30 Write block 170 len 28672 data: 9 8 9 9 13 12 8 23/30 Read block 327 len 28672 24/30 Read block 81 len 16384 25/30 Read block 121 len 8192 26/30 Read block 251 len 16384 27/30 Write block 223 len 36864 data: 15 13 12 9 9 7 7 6 9 28/30 Read block 280 len 8192 29/30 Write block 298 len 40960 data: 7 6 10 9 8 10 9 7 8 9 30/30 Write block 265 len 20480 data: 10 13 13 17 15 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:43.460911018Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1752} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 223 223 223 F F F 1 020-039 138 138 138 223 223 223 F F F 2 040-059 138 138 138 223 223 223 F F F 3 060-079 140 140 140 226 226 226 F F F 4 080-099 140 140 140 226 226 226 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 136 136 136 221 221 221 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 136 136 136 221 221 221 T T T 9 180-199 138 138 138 223 223 223 F F F 10 200-219 138 138 138 223 223 223 F F F 11 220-239 138 138 138 223 223 223 T T T 12 240-259 136 136 136 221 221 221 F F F 13 260-279 140 140 140 226 226 226 T T T 14 280-299 138 138 138 224 224 224 T T T 15 300-319 140 140 140 226 226 226 T T T 16 320-339 140 140 140 226 226 226 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 137 137 137 222 222 222 F F F 19 380-399 140 140 140 226 226 226 F F F 20 400-419 138 138 138 223 223 223 F F F 21 420-439 134 134 134 215 215 215 F F F 22 440-459 138 138 138 224 224 224 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 138 138 138 224 224 224 F F F 25 500-519 140 140 140 226 226 226 F F F 26 520-539 140 140 140 226 226 226 F F F 27 540-559 138 138 138 223 223 223 F F F 28 560-579 134 134 134 212 212 212 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 140, Max flush: 226 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 223 223 223 F F F 1 020-039 138 138 138 223 223 223 F F F 2 040-059 138 138 138 223 223 223 F F F 3 060-079 140 140 140 226 226 226 F F F 4 080-099 140 140 140 226 226 226 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 136 136 136 221 221 221 F F F 7 140-159 136 136 136 221 221 221 F F F 8 160-179 141 141 141 227 227 227 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 138 138 138 223 223 223 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 136 136 136 221 221 221 F F F 13 260-279 141 141 141 227 227 227 F F F 14 280-299 141 141 141 227 227 227 F F F 15 300-319 141 141 141 227 227 227 F F F 16 320-339 140 140 140 226 226 226 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 137 137 137 222 222 222 F F F 19 380-399 140 140 140 226 226 226 F F F 20 400-419 138 138 138 223 223 223 F F F 21 420-439 134 134 134 215 215 215 F F F 22 440-459 138 138 138 224 224 224 F F F 23 460-479 132 132 132 209 209 209 F F F 24 480-499 138 138 138 224 224 224 F F F 25 500-519 140 140 140 226 226 226 F F F 26 520-539 140 140 140 226 226 226 F F F 27 540-559 138 138 138 223 223 223 F F F 28 560-579 134 134 134 212 212 212 F F F 29 580-599 131 131 131 208 208 208 F F F Max gen: 141, Max flush: 227 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.729677344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.773193362Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.773823825Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.773857952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.77387696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.773893158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760} {"msg":"Crucible 5cb3f9f9-95a6-409e-b992-d24294e97afe has session id: c8ba0b4c-207a-47af-83a1-03c1b7f61748","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.774422213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Upstairs opts: Upstairs UUID: 5cb3f9f9-95a6-409e-b992-d24294e97afe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.774466228Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Crucible stats registered with UUID: 5cb3f9f9-95a6-409e-b992-d24294e97afe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.774486806Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:47.774503294Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.77501862Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775047897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760} {"msg":"5cb3f9f9-95a6-409e-b992-d24294e97afe active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775068485Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775083813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"0","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.77511622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"0","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775138587Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775170614Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.77520628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"2","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775236017Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"2","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775262404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775357144Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775383382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"ds_connection connected from Ok(127.0.0.1:53900)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775409229Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"ds_connection connected from Ok(127.0.0.1:49742)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775435126Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:47.775461533Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:47.775485981Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:47.775510188Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.775642514Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.77606588Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"2","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.776103536Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"0","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.776661627Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"0","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.776803132Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"2","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.777241206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:47.777325518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.776499878Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"ds_connection connected from Ok(127.0.0.1:38461)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.776601867Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.777679644Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.778320117Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[0]R flush_numbers[0..12]: [223, 223, 223, 226, 226, 215, 221, 221, 227, 223, 223, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.778930233Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[0]R generation[0..12]: [138, 138, 138, 140, 140, 134, 136, 136, 141, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.7789559Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.778972818Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[1]R flush_numbers[0..12]: [223, 223, 223, 226, 226, 215, 221, 221, 227, 223, 223, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.778987367Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[1]R generation[0..12]: [138, 138, 138, 140, 140, 134, 136, 136, 141, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779001525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779015514Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[2]R flush_numbers[0..12]: [223, 223, 223, 226, 226, 215, 221, 221, 227, 223, 223, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779028962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[2]R generation[0..12]: [138, 138, 138, 140, 140, 134, 136, 136, 141, 138, 138, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.77904737Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779061419Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779074138Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779086956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779099325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779111814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"":"downstairs","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779124702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779136481Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"0","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.7791497Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"1","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779162288Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"client":"2","":"downstairs","session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779175377Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779187276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"5cb3f9f9-95a6-409e-b992-d24294e97afe is now active with session: c8ba0b4c-207a-47af-83a1-03c1b7f61748","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779199954Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779211673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760,"session_id":"c8ba0b4c-207a-47af-83a1-03c1b7f61748"} {"msg":"The guest has finished waiting for activation with:142"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.779232091Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760} {"msg":"Activated sub_volume 5cb3f9f9-95a6-409e-b992-d24294e97afe","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:25:57.77924318Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 315 len 4096 data: 14 02/30 Write block 562 len 24576 data: 19 17 15 13 13 11 03/30 Flush 04/30 Write block 492 len 24576 data: 11 12 13 10 9 9 05/30 Read block 142 len 16384 06/30 Read block 440 len 20480 07/30 Read block 340 len 24576 08/30 Write block 328 len 12288 data: 9 8 12 09/30 Read block 346 len 20480 10/30 Read block 356 len 40960 11/30 Write block 422 len 24576 data: 12 12 10 13 13 16 12/30 Read block 556 len 24576 13/30 Read block 14 len 16384 14/30 Read block 77 len 32768 15/30 Read block 248 len 16384 16/30 Write block 178 len 4096 data: 7 17/30 Write block 466 len 32768 data: 15 15 16 17 20 20 16 16 18/30 Write block 144 len 40960 data: 14 12 12 11 13 13 13 12 12 14 19/30 Write block 535 len 28672 data: 11 9 10 11 9 7 9 20/30 Write block 300 len 28672 data: 11 10 9 11 10 8 9 21/30 Write block 270 len 8192 data: 17 17 22/30 Flush 23/30 Write block 369 len 32768 data: 10 10 10 10 10 9 8 10 24/30 Write block 268 len 28672 data: 18 16 18 18 17 14 12 25/30 Write block 158 len 40960 data: 12 10 11 10 12 16 13 11 12 12 26/30 Read block 139 len 4096 27/30 Read block 340 len 20480 28/30 Write block 207 len 36864 data: 7 9 10 10 11 11 14 15 15 29/30 Write block 594 len 8192 data: 11 11 30/30 Read block 289 len 12288 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 8 7 6 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:25:57.799361176Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1760} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 223 223 223 F F F 1 020-039 138 138 138 223 223 223 F F F 2 040-059 138 138 138 223 223 223 F F F 3 060-079 140 140 140 226 226 226 F F F 4 080-099 140 140 140 226 226 226 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 136 136 136 221 221 221 F F F 7 140-159 142 142 142 229 229 229 T T T 8 160-179 142 142 142 229 229 229 T T T 9 180-199 138 138 138 223 223 223 F F F 10 200-219 138 138 138 223 223 223 T T T 11 220-239 141 141 141 227 227 227 F F F 12 240-259 136 136 136 221 221 221 F F F 13 260-279 142 142 142 229 229 229 T T T 14 280-299 141 141 141 227 227 227 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 142 142 142 229 229 229 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 137 137 137 222 222 222 T T T 19 380-399 140 140 140 226 226 226 F F F 20 400-419 138 138 138 223 223 223 F F F 21 420-439 142 142 142 229 229 229 F F F 22 440-459 138 138 138 224 224 224 F F F 23 460-479 142 142 142 229 229 229 F F F 24 480-499 142 142 142 229 229 229 F F F 25 500-519 140 140 140 226 226 226 F F F 26 520-539 142 142 142 229 229 229 F F F 27 540-559 142 142 142 229 229 229 F F F 28 560-579 142 142 142 228 228 228 F F F 29 580-599 131 131 131 208 208 208 T T T Max gen: 142, Max flush: 229 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 223 223 223 F F F 1 020-039 138 138 138 223 223 223 F F F 2 040-059 138 138 138 223 223 223 F F F 3 060-079 140 140 140 226 226 226 F F F 4 080-099 140 140 140 226 226 226 F F F 5 100-119 134 134 134 215 215 215 F F F 6 120-139 136 136 136 221 221 221 F F F 7 140-159 143 143 143 230 230 230 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 143 143 143 230 230 230 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 136 136 136 221 221 221 F F F 13 260-279 143 143 143 230 230 230 F F F 14 280-299 141 141 141 227 227 227 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 142 142 142 229 229 229 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 140 140 140 226 226 226 F F F 20 400-419 138 138 138 223 223 223 F F F 21 420-439 142 142 142 229 229 229 F F F 22 440-459 138 138 138 224 224 224 F F F 23 460-479 142 142 142 229 229 229 F F F 24 480-499 142 142 142 229 229 229 F F F 25 500-519 140 140 140 226 226 226 F F F 26 520-539 142 142 142 229 229 229 F F F 27 540-559 142 142 142 229 229 229 F F F 28 560-579 142 142 142 228 228 228 F F F 29 580-599 143 143 143 230 230 230 F F F Max gen: 143, Max flush: 230 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.046352624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:12.088217496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.088597906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.119576211Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.120124744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.12016095Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.120182587Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.120200026Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768} {"msg":"Crucible 6a429a82-bffc-48ab-b32c-6535cb2ff3a3 has session id: 01cd663d-6e0a-4d14-b087-82f7c339e12f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.120717761Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Upstairs opts: Upstairs UUID: 6a429a82-bffc-48ab-b32c-6535cb2ff3a3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.120762197Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Crucible stats registered with UUID: 6a429a82-bffc-48ab-b32c-6535cb2ff3a3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.120784424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:12.120801402Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121288881Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121313359Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768} {"msg":"6a429a82-bffc-48ab-b32c-6535cb2ff3a3 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121363774Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121390421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"0","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121422247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"0","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121438506Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"1","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121453394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"1","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121467543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"2","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121521237Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"2","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121553954Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.12158282Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121612457Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51296)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121639894Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45209)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121665152Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47961)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.121690839Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.122330322Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"1","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.12254098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"0","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.122577446Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"2","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.122949047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"1","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123025529Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"2","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123259724Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"0","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123545174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.12358525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[0]R flush_numbers[0..12]: [223, 223, 223, 226, 226, 215, 221, 230, 230, 223, 230, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123791259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[0]R generation[0..12]: [138, 138, 138, 140, 140, 134, 136, 143, 143, 138, 143, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123827385Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123853882Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[1]R flush_numbers[0..12]: [223, 223, 223, 226, 226, 215, 221, 230, 230, 223, 230, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123879979Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[1]R generation[0..12]: [138, 138, 138, 140, 140, 134, 136, 143, 143, 138, 143, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123903447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123928634Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[2]R flush_numbers[0..12]: [223, 223, 223, 226, 226, 215, 221, 230, 230, 223, 230, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123955911Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[2]R generation[0..12]: [138, 138, 138, 140, 140, 134, 136, 143, 143, 138, 143, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.123984178Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124011115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124037533Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-03-18T15:26:12.12406336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124091827Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124113065Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"":"downstairs","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124141912Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124165949Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"0","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124191396Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"1","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124216814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"client":"2","":"downstairs","session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124242391Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124267129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"6a429a82-bffc-48ab-b32c-6535cb2ff3a3 is now active with session: 01cd663d-6e0a-4d14-b087-82f7c339e12f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124292406Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:26:12.124316323Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768,"session_id":"01cd663d-6e0a-4d14-b087-82f7c339e12f"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124342401Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768} {"msg":"Activated sub_volume 6a429a82-bffc-48ab-b32c-6535cb2ff3a3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:12.124361399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 102 len 16384 data: 17 16 14 10 02/30 Read block 439 len 36864 03/30 Flush 04/30 Write block 279 len 36864 data: 12 11 12 15 16 17 17 17 15 05/30 Write block 578 len 40960 data: 11 10 10 14 13 12 15 12 15 15 06/30 Read block 420 len 16384 07/30 Write block 492 len 32768 data: 12 13 14 11 10 10 8 8 08/30 Write block 107 len 8192 data: 9 10 09/30 Write block 473 len 20480 data: 17 12 8 9 7 10/30 Write block 38 len 4096 data: 10 11/30 Write block 423 len 20480 data: 13 11 14 14 17 12/30 Write block 560 len 36864 data: 19 21 20 18 16 14 14 12 10 13/30 Write block 513 len 8192 data: 15 13 14/30 Write block 415 len 16384 data: 12 13 14 11 15/30 Write block 139 len 36864 data: 15 15 16 16 16 15 13 13 12 16/30 Read block 560 len 12288 17/30 Write block 143 len 36864 data: 17 16 14 14 13 14 14 14 13 18/30 Read block 378 len 12288 19/30 Read block 332 len 12288 20/30 Read block 492 len 16384 21/30 Flush 22/30 Write block 117 len 32768 data: 7 8 10 9 11 11 12 11 23/30 Flush 24/30 Flush 25/30 Write block 555 len 24576 data: 14 14 14 16 13 20 26/30 Read block 121 len 4096 27/30 Read block 310 len 16384 28/30 Write block 246 len 32768 data: 10 12 12 11 11 11 14 11 29/30 Read block 249 len 32768 30/30 Write block 50 len 20480 data: 14 12 14 13 13 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:12.144983722Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1768} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 223 223 223 F F F 1 020-039 144 144 144 232 232 232 F F F 2 040-059 138 138 138 223 223 223 T T T 3 060-079 140 140 140 226 226 226 F F F 4 080-099 140 140 140 226 226 226 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 144 144 144 233 233 233 F F F 7 140-159 144 144 144 232 232 232 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 143 143 143 230 230 230 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 136 136 136 221 221 221 T T T 13 260-279 144 144 144 232 232 232 F F F 14 280-299 144 144 144 232 232 232 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 142 142 142 229 229 229 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 140 140 140 226 226 226 F F F 20 400-419 144 144 144 232 232 232 F F F 21 420-439 144 144 144 232 232 232 F F F 22 440-459 138 138 138 224 224 224 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 144 144 144 232 232 232 F F F 25 500-519 144 144 144 232 232 232 F F F 26 520-539 142 142 142 229 229 229 F F F 27 540-559 142 142 142 229 229 229 T T T 28 560-579 144 144 144 232 232 232 T T T 29 580-599 144 144 144 232 232 232 F F F Max gen: 144, Max flush: 233 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 138 223 223 223 F F F 1 020-039 144 144 144 232 232 232 F F F 2 040-059 145 145 145 234 234 234 F F F 3 060-079 140 140 140 226 226 226 F F F 4 080-099 140 140 140 226 226 226 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 144 144 144 233 233 233 F F F 7 140-159 144 144 144 232 232 232 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 143 143 143 230 230 230 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 145 145 145 234 234 234 F F F 13 260-279 144 144 144 232 232 232 F F F 14 280-299 144 144 144 232 232 232 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 142 142 142 229 229 229 F F F 17 340-359 129 129 129 204 204 204 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 140 140 140 226 226 226 F F F 20 400-419 144 144 144 232 232 232 F F F 21 420-439 144 144 144 232 232 232 F F F 22 440-459 138 138 138 224 224 224 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 144 144 144 232 232 232 F F F 25 500-519 144 144 144 232 232 232 F F F 26 520-539 142 142 142 229 229 229 F F F 27 540-559 145 145 145 234 234 234 F F F 28 560-579 145 145 145 234 234 234 F F F 29 580-599 144 144 144 232 232 232 F F F Max gen: 145, Max flush: 234 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.403949437Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.447953804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.448665209Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.448698796Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.448712424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.448726593Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776} {"msg":"Crucible 843635e1-37c6-4acc-8a45-9e59fa403bdf has session id: dbab9b0c-6016-4ac6-88d1-4869ca9730a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449243408Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Upstairs opts: Upstairs UUID: 843635e1-37c6-4acc-8a45-9e59fa403bdf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449300752Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Crucible stats registered with UUID: 843635e1-37c6-4acc-8a45-9e59fa403bdf","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.44932608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:16.449348817Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449757784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449781042Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776} {"msg":"843635e1-37c6-4acc-8a45-9e59fa403bdf active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449850175Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449867713Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"0","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.44989611Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"0","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449909938Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"1","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449923817Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"1","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449936516Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"2","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.449955004Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"2","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.450030646Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.450063402Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.45008449Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44858)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.450111887Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:33747)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.450136795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57756)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.450301667Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.450797395Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"2","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.450824352Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"0","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.451461125Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"2","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.451593662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"0","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.451927816Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"1","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.452146913Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.452234734Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.452573668Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"1","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[0]R flush_numbers[0..12]: [223, 232, 234, 226, 226, 233, 233, 232, 230, 223, 230, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453069396Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[0]R generation[0..12]: [138, 144, 145, 140, 140, 144, 144, 144, 143, 138, 143, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453087725Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453102993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[1]R flush_numbers[0..12]: [223, 232, 234, 226, 226, 233, 233, 232, 230, 223, 230, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453116571Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[1]R generation[0..12]: [138, 144, 145, 140, 140, 144, 144, 144, 143, 138, 143, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.45312977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453143649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[2]R flush_numbers[0..12]: [223, 232, 234, 226, 226, 233, 233, 232, 230, 223, 230, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453157577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[2]R generation[0..12]: [138, 144, 145, 140, 140, 144, 144, 144, 143, 138, 143, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453174265Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453188324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453201043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:26:16.453215391Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453293723Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453308921Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"":"downstairs","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.45332215Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453334239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"0","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453347087Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"1","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453359746Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"client":"2","":"downstairs","session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453372415Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453384113Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"843635e1-37c6-4acc-8a45-9e59fa403bdf is now active with session: dbab9b0c-6016-4ac6-88d1-4869ca9730a6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453396492Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.453407911Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776,"session_id":"dbab9b0c-6016-4ac6-88d1-4869ca9730a6"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:16.45341973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776} {"msg":"Activated sub_volume 843635e1-37c6-4acc-8a45-9e59fa403bdf","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:26:16.453430678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 587 len 8192 data: 16 13 02/30 Flush 03/30 Write block 492 len 36864 data: 13 14 15 12 11 11 9 9 10 04/30 Write block 216 len 16384 data: 17 19 17 19 05/30 Write block 333 len 40960 data: 14 13 16 14 15 18 18 16 17 14 06/30 Read block 104 len 36864 07/30 Write block 447 len 20480 data: 12 11 13 12 14 08/30 Read block 492 len 20480 09/30 Write block 203 len 16384 data: 10 7 9 8 10/30 Write block 56 len 24576 data: 14 12 12 11 10 9 11/30 Read block 596 len 4096 12/30 Write block 345 len 12288 data: 9 8 8 13/30 Read block 496 len 36864 14/30 Flush 15/30 Write block 397 len 28672 data: 14 14 15 15 14 16 17 16/30 Write block 443 len 12288 data: 9 10 9 17/30 Write block 56 len 12288 data: 15 13 13 18/30 Read block 123 len 4096 19/30 Read block 316 len 36864 20/30 Write block 6 len 24576 data: 11 11 11 9 9 9 21/30 Flush 22/30 Write block 381 len 28672 data: 8 6 8 9 10 7 10 23/30 Flush 24/30 Write block 416 len 12288 data: 14 15 12 25/30 Flush 26/30 Write block 489 len 36864 data: 10 8 7 14 15 16 13 12 12 27/30 Read block 77 len 28672 28/30 Read block 50 len 12288 29/30 Write block 529 len 16384 data: 11 10 9 10 30/30 Read block 276 len 12288 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 5 3 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:16.47321507Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1776} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 237 237 237 F F F 1 020-039 144 144 144 232 232 232 F F F 2 040-059 146 146 146 237 237 237 F F F 3 060-079 146 146 146 236 236 236 F F F 4 080-099 140 140 140 226 226 226 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 144 144 144 233 233 233 F F F 7 140-159 144 144 144 232 232 232 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 146 146 146 236 236 236 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 145 145 145 234 234 234 F F F 13 260-279 144 144 144 232 232 232 F F F 14 280-299 144 144 144 232 232 232 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 146 146 146 236 236 236 F F F 17 340-359 146 146 146 236 236 236 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 146 146 146 238 238 238 F F F 20 400-419 146 146 146 239 239 239 F F F 21 420-439 144 144 144 232 232 232 F F F 22 440-459 146 146 146 237 237 237 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 146 146 146 236 236 236 T T T 25 500-519 146 146 146 236 236 236 F F F 26 520-539 142 142 142 229 229 229 T T T 27 540-559 145 145 145 234 234 234 F F F 28 560-579 145 145 145 234 234 234 F F F 29 580-599 146 146 146 235 235 235 F F F Max gen: 146, Max flush: 239 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 237 237 237 F F F 1 020-039 144 144 144 232 232 232 F F F 2 040-059 146 146 146 237 237 237 F F F 3 060-079 146 146 146 236 236 236 F F F 4 080-099 140 140 140 226 226 226 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 144 144 144 233 233 233 F F F 7 140-159 144 144 144 232 232 232 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 146 146 146 236 236 236 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 145 145 145 234 234 234 F F F 13 260-279 144 144 144 232 232 232 F F F 14 280-299 144 144 144 232 232 232 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 146 146 146 236 236 236 F F F 17 340-359 146 146 146 236 236 236 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 146 146 146 238 238 238 F F F 20 400-419 146 146 146 239 239 239 F F F 21 420-439 144 144 144 232 232 232 F F F 22 440-459 146 146 146 237 237 237 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 147 147 147 240 240 240 F F F 25 500-519 146 146 146 236 236 236 F F F 26 520-539 147 147 147 240 240 240 F F F 27 540-559 145 145 145 234 234 234 F F F 28 560-579 145 145 145 234 234 234 F F F 29 580-599 146 146 146 235 235 235 F F F Max gen: 147, Max flush: 240 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.693570673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:20.736129892Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.736481345Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.767254702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.76784866Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.767882256Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.767896435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.767915782Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784} {"msg":"Crucible 9b3b231c-9ded-477d-ad31-4fb471a5364c has session id: f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.768507Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Upstairs opts: Upstairs UUID: 9b3b231c-9ded-477d-ad31-4fb471a5364c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.768561985Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Crucible stats registered with UUID: 9b3b231c-9ded-477d-ad31-4fb471a5364c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.768595121Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:20.768612319Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769042664Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769061582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784} {"msg":"9b3b231c-9ded-477d-ad31-4fb471a5364c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769134254Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769151543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"0","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.7691797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"0","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769193628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"1","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769207587Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"1","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769220305Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"2","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769233894Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"2","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769259771Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769288218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769314875Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36891)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769348032Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33195)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769480018Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36030)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.769625423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.770104612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"2","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.770254657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"1","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.770291083Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"0","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.770729267Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"2","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.770766773Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"1","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.77098062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"0","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771306076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771391597Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[0]R flush_numbers[0..12]: [237, 232, 237, 236, 226, 233, 233, 232, 230, 223, 236, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771569659Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[0]R generation[0..12]: [146, 144, 146, 146, 140, 144, 144, 144, 143, 138, 146, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771601365Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771617264Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[1]R flush_numbers[0..12]: [237, 232, 237, 236, 226, 233, 233, 232, 230, 223, 236, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771630992Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[1]R generation[0..12]: [146, 144, 146, 146, 140, 144, 144, 144, 143, 138, 146, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771644491Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771657979Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[2]R flush_numbers[0..12]: [237, 232, 237, 236, 226, 233, 233, 232, 230, 223, 236, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771671318Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[2]R generation[0..12]: [146, 144, 146, 146, 140, 144, 144, 144, 143, 138, 146, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771684557Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771698005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771710984Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771723713Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:26:20.771736291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771755169Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"":"downstairs","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771767838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771779347Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"0","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771794655Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"1","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771807364Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"client":"2","":"downstairs","session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771820052Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771831861Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"9b3b231c-9ded-477d-ad31-4fb471a5364c is now active with session: f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.7718442Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771855859Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784,"session_id":"f7e0ffec-4be7-46e7-86fb-e7317a5f1a4e"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-18T15:26:20.771867847Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784} {"msg":"Activated sub_volume 9b3b231c-9ded-477d-ad31-4fb471a5364c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:20.771883136Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 424 len 12288 data: 12 15 15 02/30 Flush 03/30 Write block 430 len 32768 data: 21 21 19 16 15 14 12 10 04/30 Flush 05/30 Write block 577 len 12288 data: 9 12 11 06/30 Write block 90 len 16384 data: 6 7 7 7 07/30 Read block 509 len 12288 08/30 Read block 393 len 20480 09/30 Read block 307 len 28672 10/30 Read block 285 len 4096 11/30 Write block 54 len 28672 data: 14 14 16 14 14 12 11 12/30 Write block 127 len 8192 data: 8 10 13/30 Flush 14/30 Write block 263 len 36864 data: 9 8 11 14 14 19 17 19 19 15/30 Read block 428 len 20480 16/30 Write block 212 len 32768 data: 12 15 16 16 18 20 18 20 17/30 Read block 189 len 8192 18/30 Write block 241 len 8192 data: 9 10 19/30 Read block 296 len 4096 20/30 Write block 322 len 12288 data: 8 10 12 21/30 Write block 560 len 20480 data: 21 22 21 19 17 22/30 Flush 23/30 Write block 147 len 40960 data: 14 15 15 15 14 13 15 13 11 11 24/30 Read block 182 len 32768 25/30 Read block 282 len 12288 26/30 Read block 58 len 16384 27/30 Write block 519 len 4096 data: 15 28/30 Read block 588 len 4096 29/30 Read block 590 len 32768 30/30 Write block 579 len 20480 data: 12 11 15 14 13 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 10 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:20.791058941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1784} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 237 237 237 F F F 1 020-039 144 144 144 232 232 232 F F F 2 040-059 148 148 148 243 243 243 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 148 148 148 243 243 243 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 148 148 148 243 243 243 F F F 7 140-159 144 144 144 232 232 232 T T T 8 160-179 143 143 143 230 230 230 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 148 148 148 244 244 244 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 148 148 148 244 244 244 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 144 144 144 232 232 232 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 148 148 148 244 244 244 F F F 17 340-359 146 146 146 236 236 236 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 146 146 146 238 238 238 F F F 20 400-419 146 146 146 239 239 239 F F F 21 420-439 148 148 148 242 242 242 F F F 22 440-459 146 146 146 237 237 237 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 147 147 147 240 240 240 F F F 25 500-519 146 146 146 236 236 236 T T T 26 520-539 147 147 147 240 240 240 F F F 27 540-559 145 145 145 234 234 234 F F F 28 560-579 148 148 148 244 244 244 T T T 29 580-599 146 146 146 235 235 235 T T T Max gen: 148, Max flush: 244 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 237 237 237 F F F 1 020-039 144 144 144 232 232 232 F F F 2 040-059 148 148 148 243 243 243 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 148 148 148 243 243 243 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 148 148 148 243 243 243 F F F 7 140-159 149 149 149 245 245 245 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 138 138 138 223 223 223 F F F 10 200-219 148 148 148 244 244 244 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 148 148 148 244 244 244 F F F 13 260-279 148 148 148 244 244 244 F F F 14 280-299 144 144 144 232 232 232 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 148 148 148 244 244 244 F F F 17 340-359 146 146 146 236 236 236 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 146 146 146 238 238 238 F F F 20 400-419 146 146 146 239 239 239 F F F 21 420-439 148 148 148 242 242 242 F F F 22 440-459 146 146 146 237 237 237 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 147 147 147 240 240 240 F F F 25 500-519 149 149 149 245 245 245 F F F 26 520-539 147 147 147 240 240 240 F F F 27 540-559 145 145 145 234 234 234 F F F 28 560-579 149 149 149 245 245 245 F F F 29 580-599 149 149 149 245 245 245 F F F Max gen: 149, Max flush: 245 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.065785551Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.110426941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.111008Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.111043487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.111057575Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.111071864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792} {"msg":"Crucible ecad595e-7aaf-4fd3-b2fd-f486b418621c has session id: fbbf1610-60f1-48ad-8456-f0d4b31eb0fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.111619576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Upstairs opts: Upstairs UUID: ecad595e-7aaf-4fd3-b2fd-f486b418621c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.111658262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Crucible stats registered with UUID: ecad595e-7aaf-4fd3-b2fd-f486b418621c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.11168078Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:25.111696678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112166169Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112190766Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792} {"msg":"ecad595e-7aaf-4fd3-b2fd-f486b418621c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112259619Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112281427Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"0","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112313833Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"0","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112329881Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"1","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.11234479Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"1","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112364698Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"2","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112390995Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"2","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112417562Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112444129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112470997Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38095)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112497664Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:47379)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.11253282Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:51962)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.112603483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.113189441Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"1","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.113217628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"0","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.113899447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"1","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.113931443Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"0","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.114489445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.114575676Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.114959675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"2","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.115470062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"2","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[0]R flush_numbers[0..12]: [237, 232, 243, 243, 243, 233, 243, 245, 230, 223, 244, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116013305Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[0]R generation[0..12]: [146, 144, 148, 148, 148, 144, 148, 149, 143, 138, 148, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116038312Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.11605691Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[1]R flush_numbers[0..12]: [237, 232, 243, 243, 243, 233, 243, 245, 230, 223, 244, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116074118Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[1]R generation[0..12]: [146, 144, 148, 148, 148, 144, 148, 149, 143, 138, 148, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116090817Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116107755Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[2]R flush_numbers[0..12]: [237, 232, 243, 243, 243, 233, 243, 245, 230, 223, 244, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116125853Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[2]R generation[0..12]: [146, 144, 148, 148, 148, 144, 148, 149, 143, 138, 148, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116142801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116159939Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":Wait for a query_work_queue command to finish before sending IO "Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116187076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116204914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116220553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116236141Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"":"downstairs","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116251909Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116266498Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"0","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116282626Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"1","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116298415Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"client":"2","":"downstairs","session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116314303Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116328941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"ecad595e-7aaf-4fd3-b2fd-f486b418621c is now active with session: fbbf1610-60f1-48ad-8456-f0d4b31eb0fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.11634463Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116359268Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792,"session_id":"fbbf1610-60f1-48ad-8456-f0d4b31eb0fd"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116378986Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792} {"msg":"Activated sub_volume ecad595e-7aaf-4fd3-b2fd-f486b418621c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:25.116392765Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 553 len 12288 data: 15 12 15 02/30 Write block 279 len 20480 data: 13 12 13 16 17 03/30 Flush 04/30 Write block 195 len 8192 data: 10 13 05/30 Write block 47 len 40960 data: 14 13 14 15 13 15 14 15 15 17 06/30 Read block 538 len 4096 07/30 Write block 273 len 16384 data: 15 13 11 8 08/30 Read block 253 len 36864 09/30 Read block 135 len 16384 10/30 Read block 524 len 32768 11/30 Read block 149 len 4096 12/30 Read block 535 len 40960 13/30 Flush 14/30 Write block 35 len 36864 data: 12 11 12 11 8 9 12 13 13 15/30 Read block 565 len 32768 16/30 Write block 399 len 36864 data: 16 16 15 17 18 17 16 15 13 17/30 Read block 88 len 36864 18/30 Write block 423 len 40960 data: 14 13 16 16 18 16 18 22 22 20 19/30 Read block 117 len 8192 20/30 Write block 30 len 8192 data: 15 14 21/30 Write block 80 len 20480 data: 11 11 11 8 10 22/30 Read block 301 len 24576 23/30 Read block 233 len 4096 24/30 Read block 447 len 16384 25/30 Read block 305 len 8192 26/30 Flush 27/30 Write block 19 len 4096 data: 11 28/30 Write block 205 len 32768 data: 10 9 8 10 11 11 12 13 29/30 Write block 434 len 16384 data: 16 15 13 11 30/30 Read block 96 len 32768 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 2 4 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:25.137242144Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1792} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 237 237 237 T T T 1 020-039 150 150 150 248 248 248 F F F 2 040-059 150 150 150 248 248 248 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 150 150 150 248 248 248 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 148 148 148 243 243 243 F F F 7 140-159 149 149 149 245 245 245 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 150 150 150 247 247 247 F F F 10 200-219 148 148 148 244 244 244 T T T 11 220-239 141 141 141 227 227 227 F F F 12 240-259 148 148 148 244 244 244 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 150 150 150 246 246 246 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 148 148 148 244 244 244 F F F 17 340-359 146 146 146 236 236 236 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 150 150 150 248 248 248 F F F 21 420-439 150 150 150 248 248 248 T T T 22 440-459 146 146 146 237 237 237 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 147 147 147 240 240 240 F F F 25 500-519 149 149 149 245 245 245 F F F 26 520-539 147 147 147 240 240 240 F F F 27 540-559 150 150 150 246 246 246 F F F 28 560-579 149 149 149 245 245 245 F F F 29 580-599 149 149 149 245 245 245 F F F Max gen: 150, Max flush: 248 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 249 249 249 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 150 150 150 248 248 248 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 150 150 150 248 248 248 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 148 148 148 243 243 243 F F F 7 140-159 149 149 149 245 245 245 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 150 150 150 247 247 247 F F F 10 200-219 151 151 151 249 249 249 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 148 148 148 244 244 244 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 150 150 150 246 246 246 F F F 15 300-319 142 142 142 229 229 229 F F F 16 320-339 148 148 148 244 244 244 F F F 17 340-359 146 146 146 236 236 236 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 150 150 150 248 248 248 F F F 21 420-439 151 151 151 249 249 249 F F F 22 440-459 146 146 146 237 237 237 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 147 147 147 240 240 240 F F F 25 500-519 149 149 149 245 245 245 F F F 26 520-539 147 147 147 240 240 240 F F F 27 540-559 150 150 150 246 246 246 F F F 28 560-579 149 149 149 245 245 245 F F F 29 580-599 149 149 149 245 245 245 F F F Max gen: 151, Max flush: 249 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.366242378Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.412825334Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.413451108Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.413487435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.413501973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.413516492Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800} {"msg":"Crucible e4585e1c-2180-44fe-9436-fb8e310f85b2 has session id: cd7d8e63-b438-441c-a8c2-76357de872b2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414153065Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Upstairs opts: Upstairs UUID: e4585e1c-2180-44fe-9436-fb8e310f85b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.41419415Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Crucible stats registered with UUID: e4585e1c-2180-44fe-9436-fb8e310f85b2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414215848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:29.414229217Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414738233Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414761731Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800} {"msg":"e4585e1c-2180-44fe-9436-fb8e310f85b2 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414837263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414854851Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"0","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414882718Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"0","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414896587Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414911105Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414924164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"2","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414938192Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"2","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414952101Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414973828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.414997986Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:56736)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.415092666Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:29.415137731Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:29.415168858Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:29.415198645Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37496)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.415355699Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.415393424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.415894632Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"2","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.415923699Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"0","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.416541764Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"0","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.416569391Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"2","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.417171158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:29.417255039Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.417048473Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34221)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.417237563Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.418225899Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.418753703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[0]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 243, 245, 230, 247, 249, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419340832Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[0]R generation[0..12]: [151, 150, 150, 148, 150, 144, 148, 149, 143, 150, 151, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419370969Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419390427Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[1]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 243, 245, 230, 247, 249, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419408195Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[1]R generation[0..12]: [151, 150, 150, 148, 150, 144, 148, 149, 143, 150, 151, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419424973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419442321Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[2]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 243, 245, 230, 247, 249, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419459549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[2]R generation[0..12]: [151, 150, 150, 148, 150, 144, 148, 149, 143, 150, 151, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419476587Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419493556Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419509194Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123CWait for a query_work_queue command to finish before sending IO ","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419532582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.41954818Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419563988Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"":"downstairs","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419584266Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419599115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"0","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419615313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"1","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419631161Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"client":"2","":"downstairs","session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.41964703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419661838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"e4585e1c-2180-44fe-9436-fb8e310f85b2 is now active with session: cd7d8e63-b438-441c-a8c2-76357de872b2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419677286Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-18T15:26:39.419691775Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800,"session_id":"cd7d8e63-b438-441c-a8c2-76357de872b2"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419710413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800} {"msg":"Activated sub_volume e4585e1c-2180-44fe-9436-fb8e310f85b2","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:39.419724072Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 411 len 24576 data: 14 12 12 13 13 15 02/30 Flush 03/30 Write block 528 len 36864 data: 11 12 11 10 11 11 12 12 10 04/30 Read block 20 len 8192 05/30 Read block 209 len 24576 06/30 Flush 07/30 Write block 587 len 28672 data: 17 14 16 13 13 11 12 08/30 Read block 0 len 16384 09/30 Read block 344 len 40960 10/30 Read block 150 len 28672 11/30 Write block 120 len 16384 data: 10 12 12 13 12/30 Read block 6 len 36864 13/30 Read block 434 len 40960 14/30 Read block 52 len 20480 15/30 Write block 333 len 24576 data: 15 14 17 15 16 19 16/30 Write block 567 len 4096 data: 13 17/30 Flush 18/30 Write block 129 len 4096 data: 9 19/30 Read block 38 len 28672 20/30 Write block 440 len 8192 data: 10 9 21/30 Flush 22/30 Write block 298 len 40960 data: 8 7 12 11 10 12 11 9 10 10 23/30 Write block 191 len 12288 data: 11 11 11 24/30 Read block 322 len 24576 25/30 Flush 26/30 Write block 459 len 4096 data: 17 27/30 Write block 497 len 36864 data: 13 10 10 11 10 9 9 12 13 28/30 Read block 205 len 28672 29/30 Read block 323 len 20480 30/30 Read block 233 len 32768 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:39.440605678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1800} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 249 249 249 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 150 150 150 248 248 248 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 150 150 150 248 248 248 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 152 152 152 253 253 253 F F F 7 140-159 149 149 149 245 245 245 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 152 152 152 254 254 254 F F F 10 200-219 151 151 151 249 249 249 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 148 148 148 244 244 244 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 152 152 152 254 254 254 F F F 15 300-319 152 152 152 254 254 254 F F F 16 320-339 152 152 152 252 252 252 F F F 17 340-359 146 146 146 236 236 236 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 152 152 152 250 250 250 F F F 21 420-439 151 151 151 249 249 249 F F F 22 440-459 152 152 152 253 253 253 T T T 23 460-479 144 144 144 232 232 232 F F F 24 480-499 147 147 147 240 240 240 T T T 25 500-519 149 149 149 245 245 245 T T T 26 520-539 152 152 152 251 251 251 F F F 27 540-559 150 150 150 246 246 246 F F F 28 560-579 152 152 152 252 252 252 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 152, Max flush: 254 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 249 249 249 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 150 150 150 248 248 248 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 150 150 150 248 248 248 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 152 152 152 253 253 253 F F F 7 140-159 149 149 149 245 245 245 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 152 152 152 254 254 254 F F F 10 200-219 151 151 151 249 249 249 F F F 11 220-239 141 141 141 227 227 227 F F F 12 240-259 148 148 148 244 244 244 F F F 13 260-279 150 150 150 247 247 247 F F F 14 280-299 152 152 152 254 254 254 F F F 15 300-319 152 152 152 254 254 254 F F F 16 320-339 152 152 152 252 252 252 F F F 17 340-359 146 146 146 236 236 236 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 152 152 152 250 250 250 F F F 21 420-439 151 151 151 249 249 249 F F F 22 440-459 153 153 153 255 255 255 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 153 153 153 255 255 255 F F F 25 500-519 153 153 153 255 255 255 F F F 26 520-539 152 152 152 251 251 251 F F F 27 540-559 150 150 150 246 246 246 F F F 28 560-579 152 152 152 252 252 252 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 153, Max flush: 255 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.66742129Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.711854772Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.712432702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.712467948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.712481586Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.712495615Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808} {"msg":"Crucible 83d7b04f-7da0-4572-9c03-a3f4807f4f4f has session id: 9eaeb4c5-6443-41f0-a92a-b3f5f92faf96","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713011641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Upstairs opts: Upstairs UUID: 83d7b04f-7da0-4572-9c03-a3f4807f4f4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713052636Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Crucible stats registered with UUID: 83d7b04f-7da0-4572-9c03-a3f4807f4f4f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713076054Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:43.713089733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713533556Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713557363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808} {"msg":"83d7b04f-7da0-4572-9c03-a3f4807f4f4f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713601249Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713616547Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"0","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713643994Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"0","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713657593Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"1","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713671251Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"1","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.71368424Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"2","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713699518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"2","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713722386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713746293Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713771891Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"ds_connection connected from Ok(127.0.0.1:64959)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713812057Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"ds_connection connected from Ok(127.0.0.1:62975)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.713843703Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"ds_connection connected from Ok(127.0.0.1:38020)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.7138696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.714516952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"0","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.71454474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"1","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.715134108Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"0","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.715278592Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"1","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.715741904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.715866481Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.716458808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"2","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.716989063Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"2","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[0]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 253, 245, 230, 254, 249, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717518287Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[0]R generation[0..12]: [151, 150, 150, 148, 150, 144, 152, 149, 143, 152, 151, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717543934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717563312Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[1]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 253, 245, 230, 254, 249, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717580341Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[1]R generation[0..12]: [151, 150, 150, 148, 150, 144, 152, 149, 143, 152, 151, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717597189Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717614347Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[2]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 253, 245, 230, 254, 249, 227]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717631675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[2]R generation[0..12]: [151, 150, 150, 148, 150, 144, 152, 149, 143, 152, 151, 141]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717648414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717669161Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.71768537Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717701718Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Next flush: 256","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:26:43.717717716Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717736074Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"":"downstairs","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717752123Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717766731Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"0","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717782989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"1","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717798858Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"client":"2","":"downstairs","session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717815046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717830324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"83d7b04f-7da0-4572-9c03-a3f4807f4f4f is now active with session: 9eaeb4c5-6443-41f0-a92a-b3f5f92faf96","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:26:43.717845993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717863301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808,"session_id":"9eaeb4c5-6443-41f0-a92a-b3f5f92faf96"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.71787625Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808} {"msg":"Activated sub_volume 83d7b04f-7da0-4572-9c03-a3f4807f4f4f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:43.717887269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 120 len 16384 data: 11 13 13 14 02/30 Read block 264 len 12288 03/30 Write block 131 len 20480 data: 9 10 10 11 12 04/30 Read block 474 len 28672 05/30 Read block 197 len 32768 06/30 Read block 139 len 36864 07/30 Read block 298 len 20480 08/30 Write block 558 len 40960 data: 17 14 22 23 22 20 18 15 15 14 09/30 Write block 150 len 36864 data: 16 15 14 16 14 12 12 11 13 10/30 Write block 504 len 8192 data: 13 14 11/30 Read block 322 len 20480 12/30 Flush 13/30 Write block 332 len 40960 data: 12 16 15 18 16 17 20 19 17 18 14/30 Write block 532 len 36864 data: 12 12 13 13 11 11 12 10 8 15/30 Flush 16/30 Write block 222 len 12288 data: 14 16 14 17/30 Read block 347 len 12288 18/30 Read block 123 len 40960 19/30 Read block 515 len 36864 20/30 Read block 234 len 28672 21/30 Write block 202 len 20480 data: 10 11 8 11 10 22/30 Read block 117 len 20480 23/30 Read block 78 len 28672 24/30 Write block 237 len 24576 data: 10 10 11 10 10 11 25/30 Write block 410 len 24576 data: 14 15 13 13 14 14 26/30 Read block 49 len 4096 27/30 Write block 278 len 40960 data: 10 14 13 14 17 18 18 18 18 16 28/30 Read block 582 len 20480 29/30 Write block 149 len 32768 data: 16 17 16 15 17 15 13 13 30/30 Read block 261 len 24576 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 3 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 15 13 13 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:43.740101405Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1808} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 249 249 249 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 150 150 150 248 248 248 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 150 150 150 248 248 248 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 154 154 154 256 256 256 F F F 7 140-159 154 154 154 256 256 256 T T T 8 160-179 143 143 143 230 230 230 F F F 9 180-199 152 152 152 254 254 254 F F F 10 200-219 151 151 151 249 249 249 T T T 11 220-239 141 141 141 227 227 227 T T T 12 240-259 148 148 148 244 244 244 T T T 13 260-279 150 150 150 247 247 247 T T T 14 280-299 152 152 152 254 254 254 T T T 15 300-319 152 152 152 254 254 254 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 152 152 152 250 250 250 T T T 21 420-439 151 151 151 249 249 249 F F F 22 440-459 153 153 153 255 255 255 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 153 153 153 255 255 255 F F F 25 500-519 154 154 154 256 256 256 F F F 26 520-539 154 154 154 257 257 257 F F F 27 540-559 154 154 154 257 257 257 F F F 28 560-579 154 154 154 256 256 256 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 154, Max flush: 257 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 249 249 249 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 150 150 150 248 248 248 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 150 150 150 248 248 248 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 154 154 154 256 256 256 F F F 7 140-159 155 155 155 258 258 258 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 152 152 152 254 254 254 F F F 10 200-219 155 155 155 258 258 258 F F F 11 220-239 155 155 155 258 258 258 F F F 12 240-259 155 155 155 258 258 258 F F F 13 260-279 155 155 155 258 258 258 F F F 14 280-299 155 155 155 258 258 258 F F F 15 300-319 152 152 152 254 254 254 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 154 154 154 257 257 257 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 155 155 155 258 258 258 F F F 21 420-439 151 151 151 249 249 249 F F F 22 440-459 153 153 153 255 255 255 F F F 23 460-479 144 144 144 232 232 232 F F F 24 480-499 153 153 153 255 255 255 F F F 25 500-519 154 154 154 256 256 256 F F F 26 520-539 154 154 154 257 257 257 F F F 27 540-559 154 154 154 257 257 257 F F F 28 560-579 154 154 154 256 256 256 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 155, Max flush: 258 Begin loop 78 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:57.987901566Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.031258081Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.031915192Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.031950029Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.031964177Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.031983785Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816} {"msg":"Crucible f2e6303e-3963-4576-942b-b07feefeac38 has session id: e0979c83-19f0-423a-8401-cb9dd637d32f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.032587682Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Upstairs opts: Upstairs UUID: f2e6303e-3963-4576-942b-b07feefeac38, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.032627487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Crucible stats registered with UUID: f2e6303e-3963-4576-942b-b07feefeac38","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.032650225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:58.032670663Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033143823Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.03317606Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816} {"msg":"f2e6303e-3963-4576-942b-b07feefeac38 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033229234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033255071Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"0","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033300117Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"0","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033327494Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"1","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033355581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"1","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033380868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033408385Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033441062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.03346221Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033487657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:58.033518654Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:58.033547191Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:26:58.033574768Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57090)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033593296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64810)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033614404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.033637841Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.034140318Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"1","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.034166846Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"0","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.034833826Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"0","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.034860853Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"1","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.035491477Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:26:58.035519874Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.03435487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37530)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.034490355Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.035645504Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.036275778Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[0]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 256, 258, 230, 254, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.036906202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[0]R generation[0..12]: [151, 150, 150, 148, 150, 144, 154, 155, 143, 152, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.036931119Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.036947317Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[1]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 256, 258, 230, 254, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.036961636Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[1]R generation[0..12]: [151, 150, 150, 148, 150, 144, 154, 155, 143, 152, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.036975174Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.036989023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[2]R flush_numbers[0..12]: [249, 248, 248, 243, 248, 233, 256, 258, 230, 254, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037002931Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[2]R generation[0..12]: [151, 150, 150, 148, 150, 144, 154, 155, 143, 152, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.03701683Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037030568Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037047907Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037062395Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037075304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:27:08.037087852Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"":"downstairs","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037104411Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037116439Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"0","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037129508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"1","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037142237Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"client":"2","":"downstairs","session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037155005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037166864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"f2e6303e-3963-4576-942b-b07feefeac38 is now active with session: e0979c83-19f0-423a-8401-cb9dd637d32f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037179443Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037190992Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816,"session_id":"e0979c83-19f0-423a-8401-cb9dd637d32f"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.03720297Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816} {"msg":"Activated sub_volume f2e6303e-3963-4576-942b-b07feefeac38","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:08.037214239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 456 len 40960 data: 17 15 15 18 15 16 14 14 13 11 02/30 Read block 474 len 4096 03/30 Read block 427 len 4096 04/30 Flush 05/30 Write block 414 len 32768 data: 15 15 16 16 13 14 14 10 06/30 Flush 07/30 Write block 343 len 40960 data: 11 9 10 9 9 9 9 8 8 7 08/30 Write block 572 len 24576 data: 9 10 9 10 10 10 09/30 Flush 10/30 Write block 466 len 4096 data: 16 11/30 Write block 193 len 28672 data: 12 11 11 14 12 10 10 12/30 Write block 349 len 36864 data: 10 9 9 8 8 11 9 8 8 13/30 Write block 478 len 16384 data: 7 8 10 12 14/30 Write block 258 len 4096 data: 8 15/30 Write block 52 len 20480 data: 16 15 16 16 18 16/30 Write block 284 len 24576 data: 19 19 19 17 12 11 17/30 Read block 414 len 32768 18/30 Read block 22 len 40960 19/30 Read block 269 len 12288 20/30 Write block 299 len 24576 data: 8 13 12 11 13 12 21/30 Write block 92 len 8192 data: 8 8 22/30 Read block 291 len 36864 23/30 Flush 24/30 Write block 130 len 36864 data: 10 10 11 11 12 13 13 14 14 25/30 Read block 243 len 20480 26/30 Read block 59 len 8192 27/30 Write block 491 len 28672 data: 8 15 16 17 14 13 14 28/30 Read block 511 len 28672 29/30 Write block 401 len 36864 data: 16 18 19 18 17 16 14 16 13 30/30 Read block 379 len 20480 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:08.058613981Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1816} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 249 249 249 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 156 156 156 262 262 262 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 156 156 156 262 262 262 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 154 154 154 256 256 256 T T T 7 140-159 155 155 155 258 258 258 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 156 156 156 262 262 262 F F F 10 200-219 155 155 155 258 258 258 F F F 11 220-239 155 155 155 258 258 258 F F F 12 240-259 156 156 156 262 262 262 F F F 13 260-279 155 155 155 258 258 258 F F F 14 280-299 156 156 156 262 262 262 F F F 15 300-319 156 156 156 262 262 262 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 156 156 156 262 262 262 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 156 156 156 260 260 260 T T T 21 420-439 156 156 156 260 260 260 F F F 22 440-459 156 156 156 259 259 259 F F F 23 460-479 156 156 156 262 262 262 F F F 24 480-499 156 156 156 262 262 262 T T T 25 500-519 154 154 154 256 256 256 F F F 26 520-539 154 154 154 257 257 257 F F F 27 540-559 154 154 154 257 257 257 F F F 28 560-579 156 156 156 261 261 261 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 156, Max flush: 262 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 249 249 249 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 156 156 156 262 262 262 F F F 3 060-079 148 148 148 243 243 243 F F F 4 080-099 156 156 156 262 262 262 F F F 5 100-119 144 144 144 233 233 233 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 155 155 155 258 258 258 F F F 8 160-179 143 143 143 230 230 230 F F F 9 180-199 156 156 156 262 262 262 F F F 10 200-219 155 155 155 258 258 258 F F F 11 220-239 155 155 155 258 258 258 F F F 12 240-259 156 156 156 262 262 262 F F F 13 260-279 155 155 155 258 258 258 F F F 14 280-299 156 156 156 262 262 262 F F F 15 300-319 156 156 156 262 262 262 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 156 156 156 262 262 262 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 157 157 157 263 263 263 F F F 21 420-439 156 156 156 260 260 260 F F F 22 440-459 156 156 156 259 259 259 F F F 23 460-479 156 156 156 262 262 262 F F F 24 480-499 157 157 157 263 263 263 F F F 25 500-519 154 154 154 256 256 256 F F F 26 520-539 154 154 154 257 257 257 F F F 27 540-559 154 154 154 257 257 257 F F F 28 560-579 156 156 156 261 261 261 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 157, Max flush: 263 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.28765821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.331617982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.332205861Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.332246976Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.332265284Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.332283002Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824} {"msg":"Crucible 94a16589-db11-49e8-ab32-e945f3e50588 has session id: a6141e3c-0882-442d-8580-61742b1fe81a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.332845793Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Upstairs opts: Upstairs UUID: 94a16589-db11-49e8-ab32-e945f3e50588, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.332892638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Crucible stats registered with UUID: 94a16589-db11-49e8-ab32-e945f3e50588","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.332924155Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:12.332944253Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333402915Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333424342Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824} {"msg":"94a16589-db11-49e8-ab32-e945f3e50588 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333483736Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333511013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"0","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.33354227Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"0","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333558399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333581876Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333607483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"2","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.33363496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"2","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333658348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333684115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333711982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64527)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333742099Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55868)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333768096Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:12.333797493Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:12.3338262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:12.333854977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.333882644Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.334366534Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"2","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.33440285Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"0","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.33507005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"0","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.335098197Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"2","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.335652718Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:12.335689275Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.335353452Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36714)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.335435354Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.33652183Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337038905Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[0]R flush_numbers[0..12]: [249, 248, 262, 243, 262, 233, 263, 258, 230, 262, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.33765981Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[0]R generation[0..12]: [151, 150, 156, 148, 156, 144, 157, 155, 143, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337689047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337708665Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[1]R flush_numbers[0..12]: [249, 248, 262, 243, 262, 233, 263, 258, 230, 262, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337726263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[1]R generation[0..12]: [151, 150, 156, 148, 156, 144, 157, 155, 143, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337743571Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337760999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[2]R flush_numbers[0..12]: [249, 248, 262, 243, 262, 233, 263, 258, 230, 262, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337777838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[2]R generation[0..12]: [151, 150, 156, 148, 156, 144, 157, 155, 143, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337794876Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337812184Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337828072Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.3378512Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337867098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337887186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"":"downstairs","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337903695Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337918353Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"0","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337934401Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"1","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.33795031Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"client":"2","":"downstairs","session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337966218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.337986246Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"94a16589-db11-49e8-ab32-e945f3e50588 is now active with session: a6141e3c-0882-442d-8580-61742b1fe81a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.338002024Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.338016673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824,"session_id":"a6141e3c-0882-442d-8580-61742b1fe81a"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.338031561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824} {"msg":"Activated sub_volume 94a16589-db11-49e8-ab32-e945f3e50588","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:22.33804506Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 2 len 36864 data: 7 6 7 7 12 12 12 10 10 02/30 Read block 136 len 4096 03/30 Read block 314 len 40960 04/30 Flush 05/30 Write block 43 len 32768 data: 14 14 14 14 15 14 15 16 06/30 Write block 434 len 20480 data: 17 16 14 12 10 07/30 Write block 256 len 28672 data: 9 8 9 6 7 10 10 08/30 Read block 347 len 36864 09/30 Flush 10/30 Write block 287 len 12288 data: 18 13 12 11/30 Read block 521 len 12288 12/30 Write block 161 len 20480 data: 11 13 17 14 12 13/30 Read block 592 len 12288 14/30 Flush 15/30 Write block 416 len 40960 data: 17 17 14 15 15 11 13 15 14 17 16/30 Flush 17/30 Write block 104 len 32768 data: 15 11 10 10 11 8 8 8 18/30 Flush 19/30 Write block 513 len 24576 data: 16 14 14 14 17 16 20/30 Write block 64 len 24576 data: 13 14 15 15 14 15 21/30 Read block 265 len 16384 22/30 Read block 61 len 20480 23/30 Read block 433 len 12288 24/30 Flush 25/30 Flush 26/30 Write block 154 len 32768 data: 16 14 14 12 14 11 12 12 27/30 Write block 302 len 16384 data: 12 14 13 10 28/30 Write block 280 len 8192 data: 14 15 29/30 Read block 325 len 36864 30/30 Write block 478 len 40960 data: 8 9 11 13 13 12 13 14 15 14 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:22.358801759Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1824} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 264 264 264 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 158 158 158 265 265 265 F F F 3 060-079 158 158 158 269 269 269 F F F 4 080-099 156 156 156 262 262 262 F F F 5 100-119 158 158 158 268 268 268 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 155 155 155 258 258 258 T T T 8 160-179 158 158 158 266 266 266 T T T 9 180-199 156 156 156 262 262 262 F F F 10 200-219 155 155 155 258 258 258 F F F 11 220-239 155 155 155 258 258 258 F F F 12 240-259 158 158 158 265 265 265 F F F 13 260-279 158 158 158 265 265 265 F F F 14 280-299 158 158 158 266 266 266 T T T 15 300-319 156 156 156 262 262 262 T T T 16 320-339 154 154 154 257 257 257 F F F 17 340-359 156 156 156 262 262 262 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 158 158 158 267 267 267 F F F 21 420-439 158 158 158 267 267 267 F F F 22 440-459 156 156 156 259 259 259 F F F 23 460-479 156 156 156 262 262 262 T T T 24 480-499 157 157 157 263 263 263 T T T 25 500-519 158 158 158 269 269 269 F F F 26 520-539 154 154 154 257 257 257 F F F 27 540-559 154 154 154 257 257 257 F F F 28 560-579 156 156 156 261 261 261 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 158, Max flush: 269 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 264 264 264 F F F 1 020-039 150 150 150 248 248 248 F F F 2 040-059 158 158 158 265 265 265 F F F 3 060-079 158 158 158 269 269 269 F F F 4 080-099 156 156 156 262 262 262 F F F 5 100-119 158 158 158 268 268 268 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 159 159 159 270 270 270 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 156 156 156 262 262 262 F F F 10 200-219 155 155 155 258 258 258 F F F 11 220-239 155 155 155 258 258 258 F F F 12 240-259 158 158 158 265 265 265 F F F 13 260-279 158 158 158 265 265 265 F F F 14 280-299 159 159 159 270 270 270 F F F 15 300-319 159 159 159 270 270 270 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 156 156 156 262 262 262 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 158 158 158 267 267 267 F F F 21 420-439 158 158 158 267 267 267 F F F 22 440-459 156 156 156 259 259 259 F F F 23 460-479 159 159 159 270 270 270 F F F 24 480-499 159 159 159 270 270 270 F F F 25 500-519 158 158 158 269 269 269 F F F 26 520-539 154 154 154 257 257 257 F F F 27 540-559 154 154 154 257 257 257 F F F 28 560-579 156 156 156 261 261 261 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 159, Max flush: 270 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.604827055Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.648591217Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.649197303Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.64923104Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.649245068Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.649259357Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832} {"msg":"Crucible c6fea0a1-2f82-4b83-af3e-563f7a514ce6 has session id: d5192920-6113-49ca-91d1-fcd1744d263a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.64980109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Upstairs opts: Upstairs UUID: c6fea0a1-2f82-4b83-af3e-563f7a514ce6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.649837236Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Crucible stats registered with UUID: c6fea0a1-2f82-4b83-af3e-563f7a514ce6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.649854534Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:26.649868413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650385558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650416165Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832} {"msg":"c6fea0a1-2f82-4b83-af3e-563f7a514ce6 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650483798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650504416Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"0","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650532833Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"0","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650547672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"1","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650568609Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"1","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650592207Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"2","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.65065974Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"2","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650743581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650785707Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650811834Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51436)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650836821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47808)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650862819Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63529)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.650891576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.651455046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"0","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.651496072Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"2","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.652003579Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"2","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.652148193Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"0","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.652530473Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.652737471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.65341597Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"1","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.653936816Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"1","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[0]R flush_numbers[0..12]: [264, 248, 265, 269, 262, 268, 263, 270, 270, 262, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654477059Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[0]R generation[0..12]: [158, 150, 158, 158, 156, 158, 157, 159, 159, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654508056Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654527883Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[1]R flush_numbers[0..12]: [264, 248, 265, 269, 262, 268, 263, 270, 270, 262, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654545321Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[1]R generation[0..12]: [158, 150, 158, 158, 156, 158, 157, 159, 159, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.65456234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654579478Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[2]R flush_numbers[0..12]: [264, 248, 265, 269, 262, 268, 263, 270, 270, 262, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654596686Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[2]R generation[0..12]: [158, 150, 158, 158, 156, 158, 157, 159, 159, 156, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654617484Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654635032Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.65465067Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Generation requested: 160 >= found:160","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654673758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654689697Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654705325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"":"downstairs","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654721343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654736121Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"0","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.65475216Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"1","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654781077Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"client":"2","":"downstairs","session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654796555Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654808394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"c6fea0a1-2f82-4b83-af3e-563f7a514ce6 is now active with session: d5192920-6113-49ca-91d1-fcd1744d263a","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654820903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654832491Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832,"session_id":"d5192920-6113-49ca-91d1-fcd1744d263a"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.65484426Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832} {"msg":"Activated sub_volume c6fea0a1-2f82-4b83-af3e-563f7a514ce6","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:26.654855479Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 563 len 36864 data: 21 19 16 16 15 11 11 7 7 02/30 Read block 283 len 36864 03/30 Write block 548 len 16384 data: 11 12 12 12 04/30 Write block 494 len 36864 data: 18 15 14 15 11 11 12 11 10 05/30 Flush 06/30 Write block 290 len 40960 data: 9 9 12 12 10 9 7 5 9 9 07/30 Read block 96 len 32768 08/30 Write block 526 len 20480 data: 11 10 12 13 12 09/30 Read block 358 len 28672 10/30 Read block 49 len 8192 11/30 Read block 421 len 40960 12/30 Read block 361 len 12288 13/30 Read block 178 len 40960 14/30 Write block 278 len 32768 data: 11 15 15 16 18 19 20 20 15/30 Read block 441 len 8192 16/30 Write block 459 len 28672 data: 19 16 17 15 15 14 12 17/30 Read block 187 len 12288 18/30 Read block 38 len 16384 19/30 Read block 92 len 16384 20/30 Read block 543 len 28672 21/30 Write block 571 len 32768 data: 8 10 11 10 11 11 11 13 22/30 Read block 492 len 40960 23/30 Read block 129 len 8192 24/30 Write block 33 len 20480 data: 13 12 13 12 13 25/30 Write block 180 len 12288 data: 10 10 10 26/30 Read block 225 len 32768 27/30 Read block 385 len 28672 28/30 Read block 86 len 20480 29/30 Read block 358 len 36864 30/30 Read block 558 len 28672 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 10 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 3 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 25 25 25 75 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:26.678020895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1832} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 264 264 264 F F F 1 020-039 150 150 150 248 248 248 T T T 2 040-059 158 158 158 265 265 265 F F F 3 060-079 158 158 158 269 269 269 F F F 4 080-099 156 156 156 262 262 262 F F F 5 100-119 158 158 158 268 268 268 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 159 159 159 270 270 270 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 156 156 156 262 262 262 T T T 10 200-219 155 155 155 258 258 258 F F F 11 220-239 155 155 155 258 258 258 F F F 12 240-259 158 158 158 265 265 265 F F F 13 260-279 158 158 158 265 265 265 T T T 14 280-299 159 159 159 270 270 270 T T T 15 300-319 159 159 159 270 270 270 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 156 156 156 262 262 262 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 158 158 158 267 267 267 F F F 21 420-439 158 158 158 267 267 267 F F F 22 440-459 156 156 156 259 259 259 T T T 23 460-479 159 159 159 270 270 270 T T T 24 480-499 160 160 160 271 271 271 F F F 25 500-519 160 160 160 271 271 271 F F F 26 520-539 154 154 154 257 257 257 T T T 27 540-559 160 160 160 271 271 271 F F F 28 560-579 160 160 160 271 271 271 T T T 29 580-599 152 152 152 252 252 252 F F F Max gen: 160, Max flush: 271 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 264 264 264 F F F 1 020-039 161 161 161 272 272 272 F F F 2 040-059 158 158 158 265 265 265 F F F 3 060-079 158 158 158 269 269 269 F F F 4 080-099 156 156 156 262 262 262 F F F 5 100-119 158 158 158 268 268 268 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 159 159 159 270 270 270 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 161 161 161 272 272 272 F F F 10 200-219 155 155 155 258 258 258 F F F 11 220-239 155 155 155 258 258 258 F F F 12 240-259 158 158 158 265 265 265 F F F 13 260-279 161 161 161 272 272 272 F F F 14 280-299 161 161 161 272 272 272 F F F 15 300-319 159 159 159 270 270 270 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 156 156 156 262 262 262 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 158 158 158 267 267 267 F F F 21 420-439 158 158 158 267 267 267 F F F 22 440-459 161 161 161 272 272 272 F F F 23 460-479 161 161 161 272 272 272 F F F 24 480-499 160 160 160 271 271 271 F F F 25 500-519 160 160 160 271 271 271 F F F 26 520-539 161 161 161 272 272 272 F F F 27 540-559 160 160 160 271 271 271 F F F 28 560-579 161 161 161 272 272 272 F F F 29 580-599 152 152 152 252 252 252 F F F Max gen: 161, Max flush: 272 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.931978907Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.976434967Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.977202386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.977252451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.977271449Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.977306545Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840} {"msg":"Crucible 7a7eca30-01c5-4c0c-bcac-48578569eb19 has session id: 4d3502f9-4124-4612-ae1b-d894b50206ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.977862337Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Upstairs opts: Upstairs UUID: 7a7eca30-01c5-4c0c-bcac-48578569eb19, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.977912172Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Crucible stats registered with UUID: 7a7eca30-01c5-4c0c-bcac-48578569eb19","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.977939789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:30.977957827Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978425768Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978449925Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840} {"msg":"7a7eca30-01c5-4c0c-bcac-48578569eb19 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978521258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978540656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"0","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978586871Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"0","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978617537Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"1","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978648704Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"1","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978680001Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978708338Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978734685Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978756733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978779931Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:30.978807897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:30.978835045Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:30.978862992Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:51380)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978885349Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978909657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:61743)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.978945603Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.979559729Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"1","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.979596245Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"0","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.980167355Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"0","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.980199171Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"1","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.980766152Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:30.980802068Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.980106262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"ds_connection connected from Ok(127.0.0.1:52657)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.980172625Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.981245383Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.981823262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[0]R flush_numbers[0..12]: [264, 272, 265, 269, 262, 268, 263, 270, 270, 272, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.98240715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[0]R generation[0..12]: [158, 161, 158, 158, 156, 158, 157, 159, 159, 161, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982428238Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982443277Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[1]R flush_numbers[0..12]: [264, 272, 265, 269, 262, 268, 263, 270, 270, 272, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982457095Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[1]R generation[0..12]: [158, 161, 158, 158, 156, 158, 157, 159, 159, 161, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982470434Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982484082Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[2]R flush_numbers[0..12]: [264, 272, 265, 269, 262, 268, 263, 270, 270, 272, 258, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982497581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[2]R generation[0..12]: [158, 161, 158, 158, 156, 158, 157, 159, 159, 161, 155, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982517259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982531457Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982544106Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982557005Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982569433Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982581862Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"":"downstairs","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"No downstairs reconciliation required","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-03-18T15:27:40.98260272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982614549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"0","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982627417Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"1","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982640146Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"client":"2","":"downstairs","session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982652845Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982664783Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"7a7eca30-01c5-4c0c-bcac-48578569eb19 is now active with session: 4d3502f9-4124-4612-ae1b-d894b50206ff","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982677352Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982688941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.98270098Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840} {"msg":"Activated sub_volume 7a7eca30-01c5-4c0c-bcac-48578569eb19","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:40.982713948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 298 len 12288 data: 10 10 14 02/30 Read block 199 len 4096 03/30 Write block 77 len 20480 data: 12 10 10 12 12 04/30 Read block 65 len 16384 05/30 Write block 487 len 36864 data: 15 13 11 9 9 16 17 19 16 06/30 Write block 420 len 4096 data: 16 07/30 Flush 08/30 Write block 51 len 16384 data: 14 17 16 17 09/30 Read block 236 len 40960 10/30 Write block 156 len 4096 data: 15 11/30 Read block 540 len 36864 12/30 Write block 189 len 36864 data: 7 7 12 12 13 12 12 15 13 13/30 Write block 270 len 40960 data: 20 20 18 16 14 12 9 9 12 16 14/30 Read block 413 len 24576 15/30 Read block 414 len 28672 16/30 Write block 596 len 4096 data: 10 17/30 Write block 215 len 8192 data: 17 19 18/30 Write block 341 len 12288 data: 19 15 12 19/30 Write block 534 len 32768 data: 14 14 12 12 13 11 9 10 20/30 Read block 280 len 40960 21/30 Read block 60 len 36864 22/30 Write block 489 len 36864 data: 12 10 10 17 18 20 17 15 16 23/30 Write block 558 len 20480 data: 18 15 23 24 23 24/30 Write block 536 len 4096 data: 13 25/30 Read block 466 len 32768 26/30 Write block 152 len 4096 data: 16 27/30 Read block 416 len 28672 28/30 Read block 364 len 36864 29/30 Read block 136 len 8192 30/30 Write block 446 len 40960 data: 12 13 12 14 13 15 13 12 15 15 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 4 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.002737165Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003420943Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003603224Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00363804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003661808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003683935Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003706163Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003726141Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003739149Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003751338Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003763277Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003775416Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003787824Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003799913Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003811932Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003823821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003835859Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003847918Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003859877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003875015Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003887304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003899353Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003911351Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00392335Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003935199Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003946948Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003958626Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003970445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003982234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.003994043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00401556Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004031539Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004043868Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004055696Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004067605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004079464Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004093762Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004105871Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00411756Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004129448Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004141217Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004153326Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004165155Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004176914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004188862Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004200821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00421262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004224398Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004236387Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004248486Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004260535Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004272573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004284532Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004298271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00431034Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004322118Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004333937Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004345886Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004357744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004369573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004381422Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004393181Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00440475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004416558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004429247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004441016Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004452825Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004464743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004476602Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004488351Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004501849Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004513838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004525807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004537536Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004549304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004560973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004572652Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004584251Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004595799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004607548Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004619327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004631046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004642695Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004654523Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004666202Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004677821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00468962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004703248Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004715017Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004726756Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004738684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004750463Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004762252Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004773901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00478556Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004797378Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004809437Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004821386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004833374Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004845313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004857092Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004869211Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004895778Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004908077Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004920125Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004934234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004946293Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004958212Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00497016Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004981879Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.004993688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005014706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005030584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005042843Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005054501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00506635Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005078259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005090158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005101956Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005113735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005125534Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005137323Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005151431Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.00516326Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005175109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005186777Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:41.005198686Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1840,"session_id":"4d3502f9-4124-4612-ae1b-d894b50206ff"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 264 264 264 F F F 1 020-039 161 161 161 272 272 272 F F F 2 040-059 158 158 158 265 265 265 T T T 3 060-079 162 162 162 273 273 273 F F F 4 080-099 162 162 162 273 273 273 F F F 5 100-119 158 158 158 268 268 268 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 159 159 159 270 270 270 T T T 8 160-179 159 159 159 270 270 270 F F F 9 180-199 161 161 161 272 272 272 T T T 10 200-219 155 155 155 258 258 258 T T T 11 220-239 155 155 155 258 258 258 F F F 12 240-259 158 158 158 265 265 265 F F F 13 260-279 161 161 161 272 272 272 T T T 14 280-299 162 162 162 273 273 273 F F F 15 300-319 162 162 162 273 273 273 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 156 156 156 262 262 262 T T T 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 158 158 158 267 267 267 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 161 161 161 272 272 272 T T T 23 460-479 161 161 161 272 272 272 F F F 24 480-499 162 162 162 273 273 273 T T T 25 500-519 160 160 160 271 271 271 F F F 26 520-539 161 161 161 272 272 272 T T T 27 540-559 160 160 160 271 271 271 T T T 28 560-579 161 161 161 272 272 272 T T T 29 580-599 152 152 152 252 252 252 T T T Max gen: 162, Max flush: 273 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 264 264 264 F F F 1 020-039 161 161 161 272 272 272 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 162 162 162 273 273 273 F F F 4 080-099 162 162 162 273 273 273 F F F 5 100-119 158 158 158 268 268 268 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 163 163 163 274 274 274 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 163 163 163 274 274 274 F F F 10 200-219 163 163 163 274 274 274 F F F 11 220-239 155 155 155 258 258 258 F F F 12 240-259 158 158 158 265 265 265 F F F 13 260-279 163 163 163 274 274 274 F F F 14 280-299 162 162 162 273 273 273 F F F 15 300-319 162 162 162 273 273 273 F F F 16 320-339 154 154 154 257 257 257 F F F 17 340-359 163 163 163 274 274 274 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 F F F 20 400-419 158 158 158 267 267 267 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 163 163 163 274 274 274 F F F 23 460-479 161 161 161 272 272 272 F F F 24 480-499 163 163 163 274 274 274 F F F 25 500-519 160 160 160 271 271 271 F F F 26 520-539 163 163 163 274 274 274 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 163 163 163 274 274 274 F F F 29 580-599 163 163 163 274 274 274 F F F Max gen: 163, Max flush: 274 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.279272977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.322864137Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.323510169Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.323553645Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.323570733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.323588451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848} {"msg":"Crucible f17cf433-0f7b-4015-b6b1-f963b2b0b8d5 has session id: 7c695ada-45b5-466c-8384-a55867b4f2c9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324146263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Upstairs opts: Upstairs UUID: f17cf433-0f7b-4015-b6b1-f963b2b0b8d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324184329Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Crucible stats registered with UUID: f17cf433-0f7b-4015-b6b1-f963b2b0b8d5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324207746Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:55.324228614Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324627182Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.32464846Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848} {"msg":"f17cf433-0f7b-4015-b6b1-f963b2b0b8d5 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324709274Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324735581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"0","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324763508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"0","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324777636Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"1","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324795664Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"1","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324909622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324936699Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324951358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324969496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.324994624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44752)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.325022011Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58531)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.325147288Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:55.325252257Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:55.325284793Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:27:55.325302801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.325372414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.325718388Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"0","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.325758103Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"1","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.326297727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"0","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.326404895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"1","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.326873836Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:27:55.326956317Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.326722654Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33230)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.326826673Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.328033206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.328648071Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[0]R flush_numbers[0..12]: [264, 272, 274, 273, 273, 268, 263, 274, 270, 274, 274, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329269086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[0]R generation[0..12]: [158, 161, 163, 162, 162, 158, 157, 163, 159, 163, 163, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329301203Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329320421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[1]R flush_numbers[0..12]: [264, 272, 274, 273, 273, 268, 263, 274, 270, 274, 274, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329337589Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[1]R generation[0..12]: [158, 161, 163, 162, 162, 158, 157, 163, 159, 163, 163, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329354327Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329371315Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[2]R flush_numbers[0..12]: [264, 272, 274, 273, 273, 268, 263, 274, 270, 274, 274, 258]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329387954Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[2]R generation[0..12]: [158, 161, 163, 162, 162, 158, 157, 163, 159, 163, 163, 155]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329404602Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.32942145Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329437348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329453317Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329468815Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329484484Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"":"downstairs","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329501092Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329523399Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"0","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329539938Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"1","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329558326Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"client":"2","":"downstairs","session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329574714Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329589882Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"f17cf433-0f7b-4015-b6b1-f963b2b0b8d5 is now active with session: 7c695ada-45b5-466c-8384-a55867b4f2c9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329605791Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329620519Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848,"session_id":"7c695ada-45b5-466c-8384-a55867b4f2c9"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329652626Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848} {"msg":"Activated sub_volume f17cf433-0f7b-4015-b6b1-f963b2b0b8d5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:05.329667594Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 577 len 24576 data: 12 14 13 12 16 15 02/30 Read block 389 len 8192 03/30 Write block 200 len 8192 data: 13 12 04/30 Flush 05/30 Write block 335 len 12288 data: 19 17 18 06/30 Write block 593 len 8192 data: 13 12 07/30 Read block 221 len 8192 08/30 Read block 414 len 8192 09/30 Write block 315 len 24576 data: 15 11 12 10 12 10 10/30 Read block 174 len 12288 11/30 Read block 379 len 20480 12/30 Read block 557 len 36864 13/30 Write block 32 len 24576 data: 14 14 13 14 13 14 14/30 Write block 324 len 36864 data: 13 9 8 9 10 9 13 11 13 15/30 Read block 552 len 12288 16/30 Read block 81 len 36864 17/30 Read block 451 len 28672 18/30 Write block 458 len 36864 data: 16 20 17 18 16 16 15 13 17 19/30 Write block 14 len 16384 data: 11 12 12 13 20/30 Write block 229 len 8192 data: 8 7 21/30 Read block 467 len 4096 22/30 Read block 495 len 24576 23/30 Flush 24/30 Write block 271 len 8192 data: 21 19 25/30 Read block 8 len 24576 26/30 Read block 206 len 40960 27/30 Write block 392 len 20480 data: 13 13 13 14 16 28/30 Write block 193 len 20480 data: 14 13 13 16 14 29/30 Write block 106 len 16384 data: 11 11 12 9 30/30 Write block 241 len 36864 data: 11 12 13 11 11 11 13 13 12 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 5 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:05.34903298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1848} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 276 276 276 F F F 1 020-039 164 164 164 276 276 276 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 162 162 162 273 273 273 F F F 4 080-099 162 162 162 273 273 273 F F F 5 100-119 158 158 158 268 268 268 T T T 6 120-139 157 157 157 263 263 263 F F F 7 140-159 163 163 163 274 274 274 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 163 163 163 274 274 274 T T T 10 200-219 164 164 164 275 275 275 F F F 11 220-239 164 164 164 276 276 276 F F F 12 240-259 158 158 158 265 265 265 T T T 13 260-279 163 163 163 274 274 274 T T T 14 280-299 162 162 162 273 273 273 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 164 164 164 276 276 276 F F F 17 340-359 163 163 163 274 274 274 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 150 150 150 248 248 248 T T T 20 400-419 158 158 158 267 267 267 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 164 164 164 276 276 276 F F F 23 460-479 164 164 164 276 276 276 F F F 24 480-499 163 163 163 274 274 274 F F F 25 500-519 160 160 160 271 271 271 F F F 26 520-539 163 163 163 274 274 274 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 164 164 164 275 275 275 F F F 29 580-599 164 164 164 276 276 276 F F F Max gen: 164, Max flush: 276 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 276 276 276 F F F 1 020-039 164 164 164 276 276 276 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 162 162 162 273 273 273 F F F 4 080-099 162 162 162 273 273 273 F F F 5 100-119 165 165 165 277 277 277 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 163 163 163 274 274 274 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 165 165 165 277 277 277 F F F 10 200-219 164 164 164 275 275 275 F F F 11 220-239 164 164 164 276 276 276 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 162 162 162 273 273 273 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 164 164 164 276 276 276 F F F 17 340-359 163 163 163 274 274 274 F F F 18 360-379 143 143 143 230 230 230 F F F 19 380-399 165 165 165 277 277 277 F F F 20 400-419 158 158 158 267 267 267 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 164 164 164 276 276 276 F F F 23 460-479 164 164 164 276 276 276 F F F 24 480-499 163 163 163 274 274 274 F F F 25 500-519 160 160 160 271 271 271 F F F 26 520-539 163 163 163 274 274 274 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 164 164 164 275 275 275 F F F 29 580-599 164 164 164 276 276 276 F F F Max gen: 165, Max flush: 277 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.588782043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:09.630953573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.631313465Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.662219168Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.662759252Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.662799497Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.662816086Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.662833714Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856} {"msg":"Crucible d0cc9eb5-fdd2-494a-847e-dba23ca7a072 has session id: 3830f34c-6682-4482-8666-bf28ce5a1147","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.663465067Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Upstairs opts: Upstairs UUID: d0cc9eb5-fdd2-494a-847e-dba23ca7a072, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.663511082Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Crucible stats registered with UUID: d0cc9eb5-fdd2-494a-847e-dba23ca7a072","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.663541049Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:09.663567297Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664027808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664049756Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856} {"msg":"d0cc9eb5-fdd2-494a-847e-dba23ca7a072 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664116079Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664133227Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"0","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664161314Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"0","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664185802Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"1","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664212339Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"1","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664239136Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"2","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664265483Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"2","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664290991Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664319028Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664346435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"ds_connection connected from Ok(127.0.0.1:57559)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664371892Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"ds_connection connected from Ok(127.0.0.1:43368)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664398849Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"ds_connection connected from Ok(127.0.0.1:52203)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.664541534Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.665185317Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"2","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.665223633Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"1","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.665258549Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"0","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.665835848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"1","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.665974334Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"0","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.66600746Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"2","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.66639252Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666562212Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[0]R flush_numbers[0..12]: [276, 276, 274, 273, 273, 277, 263, 274, 270, 277, 275, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.66667374Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[0]R generation[0..12]: [164, 164, 163, 162, 162, 165, 157, 163, 159, 165, 164, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666692428Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666706787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[1]R flush_numbers[0..12]: [276, 276, 274, 273, 273, 277, 263, 274, 270, 277, 275, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666723735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[1]R generation[0..12]: [164, 164, 163, 162, 162, 165, 157, 163, 159, 165, 164, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666740993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666758181Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[2]R flush_numbers[0..12]: [276, 276, 274, 273, 273, 277, 263, 274, 270, 277, 275, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.66677466Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[2]R generation[0..12]: [164, 164, 163, 162, 162, 165, 157, 163, 159, 165, 164, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666791208Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666818205Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666855741Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666876719Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666905046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666934963Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"":"downstairs","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.66696587Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.666985257Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"0","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.667008365Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"1","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.667032423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"client":"2","":"downstairs","session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.667049271Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.667064459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"d0cc9eb5-fdd2-494a-847e-dba23ca7a072 is now active with session: 3830f34c-6682-4482-8666-bf28ce5a1147","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-03-18T15:28:09.667080247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.667099545Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856,"session_id":"3830f34c-6682-4482-8666-bf28ce5a1147"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.667114564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856} {"msg":"Activated sub_volume d0cc9eb5-fdd2-494a-847e-dba23ca7a072","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:09.667128113Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 481 len 4096 data: 14 02/30 Read block 20 len 8192 03/30 Read block 396 len 28672 04/30 Write block 450 len 12288 data: 14 16 14 05/30 Read block 460 len 12288 06/30 Write block 413 len 4096 data: 14 07/30 Write block 378 len 24576 data: 12 12 11 9 7 9 08/30 Write block 385 len 36864 data: 11 8 11 10 10 11 13 14 14 09/30 Write block 368 len 28672 data: 9 11 11 11 11 11 10 10/30 Read block 237 len 12288 11/30 Write block 5 len 28672 data: 8 13 13 13 11 11 10 12/30 Read block 483 len 12288 13/30 Read block 425 len 32768 14/30 Write block 26 len 16384 data: 8 7 8 12 15/30 Read block 580 len 8192 16/30 Read block 15 len 16384 17/30 Read block 144 len 4096 18/30 Write block 210 len 36864 data: 12 13 14 16 17 18 20 21 19 19/30 Write block 73 len 24576 data: 10 10 9 7 13 11 20/30 Flush 21/30 Write block 115 len 28672 data: 8 7 8 9 11 12 14 22/30 Write block 518 len 20480 data: 17 16 15 15 13 23/30 Read block 587 len 28672 24/30 Read block 71 len 4096 25/30 Read block 336 len 16384 26/30 Read block 97 len 16384 27/30 Read block 374 len 16384 28/30 Read block 162 len 24576 29/30 Write block 367 len 36864 data: 10 10 12 12 12 12 12 11 9 30/30 Read block 453 len 4096 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 7 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:09.687486914Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1856} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 166 166 166 278 278 278 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 166 166 166 278 278 278 F F F 4 080-099 162 162 162 273 273 273 F F F 5 100-119 165 165 165 277 277 277 T T T 6 120-139 157 157 157 263 263 263 T T T 7 140-159 163 163 163 274 274 274 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 165 165 165 277 277 277 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 164 164 164 276 276 276 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 162 162 162 273 273 273 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 164 164 164 276 276 276 F F F 17 340-359 163 163 163 274 274 274 F F F 18 360-379 166 166 166 278 278 278 T T T 19 380-399 166 166 166 278 278 278 F F F 20 400-419 166 166 166 278 278 278 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 166 166 166 278 278 278 F F F 23 460-479 164 164 164 276 276 276 F F F 24 480-499 166 166 166 278 278 278 F F F 25 500-519 160 160 160 271 271 271 T T T 26 520-539 163 163 163 274 274 274 T T T 27 540-559 163 163 163 274 274 274 F F F 28 560-579 164 164 164 275 275 275 F F F 29 580-599 164 164 164 276 276 276 F F F Max gen: 166, Max flush: 278 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 166 166 166 278 278 278 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 166 166 166 278 278 278 F F F 4 080-099 162 162 162 273 273 273 F F F 5 100-119 167 167 167 279 279 279 F F F 6 120-139 167 167 167 279 279 279 F F F 7 140-159 163 163 163 274 274 274 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 165 165 165 277 277 277 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 164 164 164 276 276 276 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 162 162 162 273 273 273 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 164 164 164 276 276 276 F F F 17 340-359 163 163 163 274 274 274 F F F 18 360-379 167 167 167 279 279 279 F F F 19 380-399 166 166 166 278 278 278 F F F 20 400-419 166 166 166 278 278 278 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 166 166 166 278 278 278 F F F 23 460-479 164 164 164 276 276 276 F F F 24 480-499 166 166 166 278 278 278 F F F 25 500-519 167 167 167 279 279 279 F F F 26 520-539 167 167 167 279 279 279 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 164 164 164 275 275 275 F F F 29 580-599 164 164 164 276 276 276 F F F Max gen: 167, Max flush: 279 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.953996426Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.997833571Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.998468934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.998503791Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.998518239Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.998532727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864} {"msg":"Crucible 3af116b9-4e63-475d-80fb-95f9fef7cfc8 has session id: 2193fca3-3613-475e-93d7-4a7249a45793","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999122056Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Upstairs opts: Upstairs UUID: 3af116b9-4e63-475d-80fb-95f9fef7cfc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999149243Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Crucible stats registered with UUID: 3af116b9-4e63-475d-80fb-95f9fef7cfc8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999163471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:13.99917584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999638421Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999662439Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864} {"msg":"3af116b9-4e63-475d-80fb-95f9fef7cfc8 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999725822Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999742511Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"0","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999769817Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"0","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999783666Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"1","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999798925Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"1","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999821652Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"2","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999847149Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"2","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999871297Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999901264Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999927941Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"ds_connection connected from Ok(127.0.0.1:62172)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999956968Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"ds_connection connected from Ok(127.0.0.1:43204)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:13.999982835Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"ds_connection connected from Ok(127.0.0.1:36679)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.000008163Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.000744955Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"2","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.000896559Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"0","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.0013622Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"2","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.001507445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"0","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.001982895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.002066816Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.002426509Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"1","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.002944234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"1","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[0]R flush_numbers[0..12]: [278, 278, 274, 278, 273, 279, 279, 274, 270, 277, 278, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.00346167Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[0]R generation[0..12]: [166, 166, 163, 166, 162, 167, 167, 163, 159, 165, 166, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003486247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003506165Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[1]R flush_numbers[0..12]: [278, 278, 274, 278, 273, 279, 279, 274, 270, 277, 278, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003521303Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[1]R generation[0..12]: [166, 166, 163, 166, 162, 167, 167, 163, 159, 165, 166, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003535952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.00354962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[2]R flush_numbers[0..12]: [278, 278, 274, 278, 273, 279, 279, 274, 270, 277, 278, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003562999Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[2]R generation[0..12]: [166, 166, 163, 166, 162, 167, 167, 163, 159, 165, 166, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003576418Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003590726Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003603425Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003616284Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003628852Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:28:14.003641341Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"":"downstairs","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003661019Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003673258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"0","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003686646Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"1","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003702084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"client":"2","":"downstairs","session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003715413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003727382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"3af116b9-4e63-475d-80fb-95f9fef7cfc8 is now active with session: 2193fca3-3613-475e-93d7-4a7249a45793","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003740091Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003751749Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864,"session_id":"2193fca3-3613-475e-93d7-4a7249a45793"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003763778Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 pid":1864} {"msg":"Activated sub_volume 3af116b9-4e63-475d-80fb-95f9fef7cfc8","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:14.003779116Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 4096 data: 14 02/30 Flush 03/30 Write block 520 len 36864 data: 16 16 14 11 9 10 12 11 13 04/30 Write block 388 len 40960 data: 11 11 12 14 15 15 14 15 17 15 05/30 Read block 14 len 32768 06/30 Write block 155 len 12288 data: 15 16 13 07/30 Write block 451 len 36864 data: 17 15 13 16 16 18 16 17 21 08/30 Read block 273 len 20480 09/30 Write block 363 len 36864 data: 7 6 9 10 11 11 13 13 13 10/30 Write block 510 len 40960 data: 16 18 15 17 15 15 15 18 18 17 11/30 Read block 532 len 12288 12/30 Read block 434 len 36864 13/30 Read block 42 len 28672 14/30 Write block 33 len 28672 data: 15 14 15 14 15 12 9 15/30 Write block 99 len 8192 data: 16 17 16/30 Write block 147 len 24576 data: 15 16 17 18 17 17 17/30 Read block 74 len 24576 18/30 Write block 183 len 40960 data: 12 11 11 11 9 7 8 8 13 13 19/30 Read block 368 len 36864 20/30 Write block 205 len 28672 data: 12 11 9 11 12 13 14 21/30 Read block 512 len 16384 22/30 Read block 147 len 20480 23/30 Write block 78 len 28672 data: 12 11 13 13 12 9 11 24/30 Read block 275 len 24576 25/30 Read block 30 len 24576 26/30 Read block 428 len 16384 27/30 Write block 137 len 36864 data: 15 15 16 16 17 17 18 17 15 28/30 Read block 311 len 24576 29/30 Read block 86 len 12288 30/30 Write block 284 len 8192 data: 21 21 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 9 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 7 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:14.025199396Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1864} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 166 166 166 278 278 278 T T T 2 040-059 163 163 163 274 274 274 F F F 3 060-079 166 166 166 278 278 278 T T T 4 080-099 162 162 162 273 273 273 T T T 5 100-119 167 167 167 279 279 279 T T T 6 120-139 167 167 167 279 279 279 T T T 7 140-159 163 163 163 274 274 274 T T T 8 160-179 159 159 159 270 270 270 F F F 9 180-199 165 165 165 277 277 277 T T T 10 200-219 166 166 166 278 278 278 T T T 11 220-239 164 164 164 276 276 276 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 162 162 162 273 273 273 T T T 15 300-319 164 164 164 276 276 276 F F F 16 320-339 164 164 164 276 276 276 F F F 17 340-359 163 163 163 274 274 274 F F F 18 360-379 167 167 167 279 279 279 T T T 19 380-399 166 166 166 278 278 278 T T T 20 400-419 168 168 168 280 280 280 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 166 166 166 278 278 278 T T T 23 460-479 164 164 164 276 276 276 F F F 24 480-499 166 166 166 278 278 278 F F F 25 500-519 167 167 167 279 279 279 T T T 26 520-539 167 167 167 279 279 279 T T T 27 540-559 163 163 163 274 274 274 F F F 28 560-579 164 164 164 275 275 275 F F F 29 580-599 164 164 164 276 276 276 F F F Max gen: 168, Max flush: 280 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 169 169 169 281 281 281 F F F 2 040-059 163 163 163 274 274 274 F F F 3 060-079 169 169 169 281 281 281 F F F 4 080-099 169 169 169 281 281 281 F F F 5 100-119 169 169 169 281 281 281 F F F 6 120-139 169 169 169 281 281 281 F F F 7 140-159 169 169 169 281 281 281 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 169 169 169 281 281 281 F F F 10 200-219 169 169 169 281 281 281 F F F 11 220-239 164 164 164 276 276 276 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 164 164 164 276 276 276 F F F 17 340-359 163 163 163 274 274 274 F F F 18 360-379 169 169 169 281 281 281 F F F 19 380-399 169 169 169 281 281 281 F F F 20 400-419 168 168 168 280 280 280 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 169 169 169 281 281 281 F F F 23 460-479 164 164 164 276 276 276 F F F 24 480-499 166 166 166 278 278 278 F F F 25 500-519 169 169 169 281 281 281 F F F 26 520-539 169 169 169 281 281 281 F F F 27 540-559 163 163 163 274 274 274 F F F 28 560-579 164 164 164 275 275 275 F F F 29 580-599 164 164 164 276 276 276 F F F Max gen: 169, Max flush: 281 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.307416542Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:28.349161366Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.349529548Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.38053824Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.381143766Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.381177163Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.381191152Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.38120542Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872} {"msg":"Crucible 6a7467ca-5e4b-43d2-85a1-3a2f66058830 has session id: 2c3a59b2-80f3-477a-9bcc-0a723ca5abec","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.381782949Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Upstairs opts: Upstairs UUID: 6a7467ca-5e4b-43d2-85a1-3a2f66058830, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.381826365Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Crucible stats registered with UUID: 6a7467ca-5e4b-43d2-85a1-3a2f66058830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.381851812Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:28.38186859Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382353659Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382376147Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872} {"msg":"6a7467ca-5e4b-43d2-85a1-3a2f66058830 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382432301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382458438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"0","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382487185Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"0","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382502684Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"1","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382516672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"1","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382529501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"2","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382553249Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"2","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382576276Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382612962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.38263804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"ds_connection connected from Ok(127.0.0.1:47903)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382663727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"ds_connection connected from Ok(127.0.0.1:35748)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382691444Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"ds_connection connected from Ok(127.0.0.1:41462)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.382961346Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.383312209Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"2","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.383423897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"1","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.383551973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"0","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384039022Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"1","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384065939Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"2","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384088747Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"0","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384586475Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384698993Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[0]R flush_numbers[0..12]: [278, 281, 274, 281, 281, 281, 281, 281, 270, 281, 281, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384831569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[0]R generation[0..12]: [166, 169, 163, 169, 169, 169, 169, 169, 159, 169, 169, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384852767Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384867795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[1]R flush_numbers[0..12]: [278, 281, 274, 281, 281, 281, 281, 281, 270, 281, 281, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384881924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[1]R generation[0..12]: [166, 169, 163, 169, 169, 169, 169, 169, 159, 169, 169, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384899442Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.38491379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[2]R flush_numbers[0..12]: [278, 281, 274, 281, 281, 281, 281, 281, 270, 281, 281, 276]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384928339Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[2]R generation[0..12]: [166, 169, 163, 169, 169, 169, 169, 169, 159, 169, 169, 164]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384941567Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384955096Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384967585Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384980354Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.384992662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385005221Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"":"downstairs","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385017939Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","Wait for a query_work_queue command to finish before sending IO pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385036928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"0","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385049966Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"1","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385062675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"client":"2","":"downstairs","session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385075664Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385087652Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"6a7467ca-5e4b-43d2-85a1-3a2f66058830 is now active with session: 2c3a59b2-80f3-477a-9bcc-0a723ca5abec","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385102281Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385114429Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872,"session_id":"2c3a59b2-80f3-477a-9bcc-0a723ca5abec"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385126458Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872} {"msg":"Activated sub_volume 6a7467ca-5e4b-43d2-85a1-3a2f66058830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:28.385137547Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 591 len 12288 data: 14 12 14 02/30 Write block 121 len 40960 data: 15 14 15 12 10 9 9 11 10 11 03/30 Write block 89 len 8192 data: 9 7 04/30 Write block 39 len 24576 data: 10 10 13 14 15 15 05/30 Write block 563 len 24576 data: 22 20 17 17 16 12 06/30 Flush 07/30 Write block 48 len 36864 data: 15 16 17 15 18 17 18 17 19 08/30 Read block 34 len 20480 09/30 Read block 393 len 28672 10/30 Write block 458 len 32768 data: 18 22 18 19 17 17 16 14 11/30 Read block 330 len 16384 12/30 Flush 13/30 Write block 495 len 20480 data: 18 16 17 12 12 14/30 Read block 209 len 16384 15/30 Read block 149 len 40960 16/30 Write block 389 len 16384 data: 12 13 15 16 17/30 Write block 178 len 32768 data: 8 8 11 11 11 13 12 12 18/30 Write block 325 len 28672 data: 10 9 10 11 10 14 12 19/30 Write block 3 len 32768 data: 7 8 9 14 14 14 12 12 20/30 Read block 498 len 36864 21/30 Read block 37 len 24576 22/30 Write block 322 len 16384 data: 9 11 14 11 23/30 Write block 539 len 20480 data: 12 10 11 12 15 24/30 Write block 124 len 40960 data: 13 11 10 10 12 11 12 11 12 12 25/30 Write block 11 len 20480 data: 11 12 10 12 13 26/30 Write block 233 len 28672 data: 9 11 11 11 11 11 12 27/30 Read block 253 len 16384 28/30 Write block 168 len 36864 data: 11 8 10 9 10 10 14 13 9 29/30 Read block 198 len 24576 30/30 Write block 127 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 5 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 10 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:28.404856835Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1872} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 T T T 1 020-039 170 170 170 282 282 282 F F F 2 040-059 170 170 170 283 283 283 F F F 3 060-079 169 169 169 281 281 281 F F F 4 080-099 170 170 170 282 282 282 F F F 5 100-119 169 169 169 281 281 281 F F F 6 120-139 170 170 170 282 282 282 T T T 7 140-159 169 169 169 281 281 281 F F F 8 160-179 159 159 159 270 270 270 T T T 9 180-199 169 169 169 281 281 281 T T T 10 200-219 169 169 169 281 281 281 F F F 11 220-239 164 164 164 276 276 276 T T T 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 164 164 164 276 276 276 T T T 17 340-359 163 163 163 274 274 274 F F F 18 360-379 169 169 169 281 281 281 F F F 19 380-399 169 169 169 281 281 281 T T T 20 400-419 168 168 168 280 280 280 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 170 170 170 283 283 283 F F F 23 460-479 170 170 170 283 283 283 F F F 24 480-499 166 166 166 278 278 278 T T T 25 500-519 169 169 169 281 281 281 F F F 26 520-539 169 169 169 281 281 281 T T T 27 540-559 163 163 163 274 274 274 T T T 28 560-579 170 170 170 282 282 282 F F F 29 580-599 170 170 170 282 282 282 F F F Max gen: 170, Max flush: 283 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 284 284 284 F F F 1 020-039 170 170 170 282 282 282 F F F 2 040-059 170 170 170 283 283 283 F F F 3 060-079 169 169 169 281 281 281 F F F 4 080-099 170 170 170 282 282 282 F F F 5 100-119 169 169 169 281 281 281 F F F 6 120-139 171 171 171 284 284 284 F F F 7 140-159 169 169 169 281 281 281 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 171 171 171 284 284 284 F F F 10 200-219 169 169 169 281 281 281 F F F 11 220-239 171 171 171 284 284 284 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 171 171 171 284 284 284 F F F 17 340-359 163 163 163 274 274 274 F F F 18 360-379 169 169 169 281 281 281 F F F 19 380-399 171 171 171 284 284 284 F F F 20 400-419 168 168 168 280 280 280 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 170 170 170 283 283 283 F F F 23 460-479 170 170 170 283 283 283 F F F 24 480-499 171 171 171 284 284 284 F F F 25 500-519 169 169 169 281 281 281 F F F 26 520-539 171 171 171 284 284 284 F F F 27 540-559 171 171 171 284 284 284 F F F 28 560-579 170 170 170 282 282 282 F F F 29 580-599 170 170 170 282 282 282 F F F Max gen: 171, Max flush: 284 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.693779934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:32.736410275Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.736790805Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.767622916Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.76816127Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.768199196Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.768215924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.768233132Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880} {"msg":"Crucible dc425735-ddcc-4f27-8eb8-44e0d5a76709 has session id: 7522de92-8d43-4a48-85fb-bf14f9b17ddf","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.768774975Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Upstairs opts: Upstairs UUID: dc425735-ddcc-4f27-8eb8-44e0d5a76709, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.768818721Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Crucible stats registered with UUID: dc425735-ddcc-4f27-8eb8-44e0d5a76709","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.768838329Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:32.768854007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769286211Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769325618Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880} {"msg":"dc425735-ddcc-4f27-8eb8-44e0d5a76709 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.76939451Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769421027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"0","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769451454Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"0","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769467063Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"1","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769486621Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"1","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769510628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"2","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769537225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"2","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769561793Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.76958709Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769615577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"ds_connection connected from Ok(127.0.0.1:36870)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769642764Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"ds_connection connected from Ok(127.0.0.1:46795)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769668371Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"ds_connection connected from Ok(127.0.0.1:35015)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.769695179Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.770476346Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"1","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.770519712Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"0","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.770552798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"2","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.771141617Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"2","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.771290141Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"1","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.771327707Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"0","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.771695398Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.771811616Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[0]R flush_numbers[0..12]: [284, 282, 283, 281, 282, 281, 284, 281, 284, 284, 281, 284]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.771942892Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[0]R generation[0..12]: [171, 170, 170, 169, 170, 169, 171, 169, 171, 171, 169, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.77196199Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.771976649Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[1]R flush_numbers[0..12]: [284, 282, 283, 281, 282, 281, 284, 281, 284, 284, 281, 284]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.771990818Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[1]R generation[0..12]: [171, 170, 170, 169, 170, 169, 171, 169, 171, 171, 169, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772012615Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772030683Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[2]R flush_numbers[0..12]: [284, 282, 283, 281, 282, 281, 284, 281, 284, 284, 281, 284]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772044402Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[2]R generation[0..12]: [171, 170, 170, 169, 170, 169, 171, 169, 171, 171, 169, 171]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.77205767Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772074769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:28:32.772096926Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772126803Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772148541Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772161889Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"":"downstairs","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772174688Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772186877Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"0","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772209444Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"1","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772233612Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"client":"2","":"downstairs","session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772257709Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772282377Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"dc425735-ddcc-4f27-8eb8-44e0d5a76709 is now active with session: 7522de92-8d43-4a48-85fb-bf14f9b17ddf","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772303085Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772315933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880,"session_id":"7522de92-8d43-4a48-85fb-bf14f9b17ddf"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772328152Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880} {"msg":"Activated sub_volume dc425735-ddcc-4f27-8eb8-44e0d5a76709","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:32.772339211Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 36864 data: 16 16 17 18 16 19 18 19 18 02/30 Write block 217 len 40960 data: 22 20 21 20 18 15 17 15 13 10 03/30 Flush 04/30 Write block 96 len 36864 data: 12 12 12 17 18 15 18 17 16 05/30 Read block 244 len 16384 06/30 Write block 495 len 12288 data: 19 17 18 07/30 Read block 360 len 36864 08/30 Read block 34 len 16384 09/30 Write block 3 len 12288 data: 8 9 10 10/30 Read block 362 len 36864 11/30 Read block 95 len 40960 12/30 Read block 146 len 28672 13/30 Flush 14/30 Write block 104 len 36864 data: 17 12 12 12 13 10 9 9 10 15/30 Read block 528 len 28672 16/30 Write block 487 len 20480 data: 16 14 13 11 11 17/30 Read block 28 len 40960 18/30 Read block 539 len 8192 19/30 Flush 20/30 Write block 356 len 20480 data: 9 9 7 8 8 21/30 Read block 338 len 24576 22/30 Read block 144 len 32768 23/30 Write block 568 len 32768 data: 13 12 8 9 11 12 11 12 24/30 Flush 25/30 Flush 26/30 Write block 383 len 28672 data: 10 10 12 9 12 12 13 27/30 Write block 376 len 28672 data: 11 10 13 13 12 10 8 28/30 Write block 484 len 28672 data: 14 15 16 17 15 14 12 29/30 Read block 416 len 20480 30/30 Read block 129 len 24576 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:32.794227761Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1880} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 286 286 286 F F F 1 020-039 170 170 170 282 282 282 F F F 2 040-059 172 172 172 285 285 285 F F F 3 060-079 169 169 169 281 281 281 F F F 4 080-099 172 172 172 286 286 286 F F F 5 100-119 172 172 172 287 287 287 F F F 6 120-139 171 171 171 284 284 284 F F F 7 140-159 169 169 169 281 281 281 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 171 171 171 284 284 284 F F F 10 200-219 172 172 172 285 285 285 F F F 11 220-239 172 172 172 285 285 285 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 171 171 171 284 284 284 F F F 17 340-359 172 172 172 288 288 288 F F F 18 360-379 172 172 172 288 288 288 T T T 19 380-399 171 171 171 284 284 284 T T T 20 400-419 168 168 168 280 280 280 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 170 170 170 283 283 283 F F F 23 460-479 170 170 170 283 283 283 F F F 24 480-499 172 172 172 287 287 287 T T T 25 500-519 169 169 169 281 281 281 F F F 26 520-539 171 171 171 284 284 284 F F F 27 540-559 171 171 171 284 284 284 F F F 28 560-579 172 172 172 288 288 288 F F F 29 580-599 170 170 170 282 282 282 F F F Max gen: 172, Max flush: 288 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 286 286 286 F F F 1 020-039 170 170 170 282 282 282 F F F 2 040-059 172 172 172 285 285 285 F F F 3 060-079 169 169 169 281 281 281 F F F 4 080-099 172 172 172 286 286 286 F F F 5 100-119 172 172 172 287 287 287 F F F 6 120-139 171 171 171 284 284 284 F F F 7 140-159 169 169 169 281 281 281 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 171 171 171 284 284 284 F F F 10 200-219 172 172 172 285 285 285 F F F 11 220-239 172 172 172 285 285 285 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 164 164 164 276 276 276 F F F 16 320-339 171 171 171 284 284 284 F F F 17 340-359 172 172 172 288 288 288 F F F 18 360-379 173 173 173 289 289 289 F F F 19 380-399 173 173 173 289 289 289 F F F 20 400-419 168 168 168 280 280 280 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 170 170 170 283 283 283 F F F 23 460-479 170 170 170 283 283 283 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 169 169 169 281 281 281 F F F 26 520-539 171 171 171 284 284 284 F F F 27 540-559 171 171 171 284 284 284 F F F 28 560-579 172 172 172 288 288 288 F F F 29 580-599 170 170 170 282 282 282 F F F Max gen: 173, Max flush: 289 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.054030099Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.099874213Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.100623604Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.100658761Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.100673249Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.100688537Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888} {"msg":"Crucible 40da66ab-1989-4568-ba64-d929a7f17206 has session id: ae85bff9-4a41-489c-8219-b2cce65a6690","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.101194874Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Upstairs opts: Upstairs UUID: 40da66ab-1989-4568-ba64-d929a7f17206, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.101256028Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Crucible stats registered with UUID: 40da66ab-1989-4568-ba64-d929a7f17206","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.101285655Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:37.101312272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.101768854Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.10180875Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888} {"msg":"40da66ab-1989-4568-ba64-d929a7f17206 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.101871893Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.10189721Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"0","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.101936617Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"0","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.101964934Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"1","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102048855Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"1","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102083111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"2","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102111738Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"2","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102137945Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102163263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.1021867Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"ds_connection connected from Ok(127.0.0.1:57951)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102219447Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"ds_connection connected from Ok(127.0.0.1:61342)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102304718Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"ds_connection connected from Ok(127.0.0.1:64731)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102343094Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102752501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"1","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.102792187Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"0","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.103382185Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"1","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.103493593Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"0","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.104041845Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.104167732Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.104428485Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"2","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.104915173Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"2","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[0]R flush_numbers[0..12]: [286, 282, 285, 281, 286, 287, 284, 281, 284, 284, 285, 285]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105420231Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 169, 172, 172, 171, 169, 171, 171, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105456117Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105471925Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[1]R flush_numbers[0..12]: [286, 282, 285, 281, 286, 287, 284, 281, 284, 284, 285, 285]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105490543Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 169, 172, 172, 171, 169, 171, 171, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105504542Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.10551838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[2]R flush_numbers[0..12]: [286, 282, 285, 281, 286, 287, 284, 281, 284, 284, 285, 285]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105531959Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 169, 172, 172, 171, 169, 171, 171, 172, 172]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105545547Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105559196Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105571815Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-03-18T15:28:37.105584573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105605291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.10561812Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"":"downstairs","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105630878Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105642617Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"0","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105655526Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"1","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105668164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"client":"2","":"downstairs","session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105683133Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105695142Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"40da66ab-1989-4568-ba64-d929a7f17206 is now active with session: ae85bff9-4a41-489c-8219-b2cce65a6690","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.10570765Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105719259Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888,"session_id":"ae85bff9-4a41-489c-8219-b2cce65a6690"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105731158Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888} {"msg":"Activated sub_volume 40da66ab-1989-4568-ba64-d929a7f17206","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:37.105742027Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 557 len 40960 data: 15 19 16 24 25 24 23 21 18 18 02/30 Write block 329 len 12288 data: 11 15 13 03/30 Write block 100 len 40960 data: 19 16 19 18 18 13 13 13 14 11 04/30 Write block 404 len 4096 data: 19 05/30 Write block 310 len 12288 data: 6 9 10 06/30 Read block 332 len 16384 07/30 Write block 69 len 12288 data: 16 14 13 08/30 Write block 138 len 16384 data: 16 17 17 18 09/30 Read block 434 len 20480 10/30 Write block 361 len 40960 data: 7 7 8 7 10 11 12 12 14 14 11/30 Read block 278 len 16384 12/30 Read block 178 len 12288 13/30 Flush 14/30 Write block 392 len 16384 data: 17 16 15 16 15/30 Read block 382 len 12288 16/30 Read block 340 len 24576 17/30 Write block 539 len 20480 data: 13 11 12 13 16 18/30 Write block 370 len 16384 data: 15 14 13 13 19/30 Write block 551 len 4096 data: 13 20/30 Write block 526 len 12288 data: 13 12 14 21/30 Flush 22/30 Write block 195 len 20480 data: 14 17 15 11 11 23/30 Flush 24/30 Write block 235 len 16384 data: 12 12 12 12 25/30 Read block 250 len 32768 26/30 Read block 50 len 20480 27/30 Write block 458 len 16384 data: 19 23 19 20 28/30 Read block 16 len 4096 29/30 Read block 1 len 32768 30/30 Write block 534 len 40960 data: 15 15 14 13 14 14 12 13 14 17 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:37.12503666Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1888} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 286 286 286 F F F 1 020-039 170 170 170 282 282 282 F F F 2 040-059 172 172 172 285 285 285 F F F 3 060-079 174 174 174 290 290 290 F F F 4 080-099 172 172 172 286 286 286 F F F 5 100-119 174 174 174 290 290 290 F F F 6 120-139 174 174 174 290 290 290 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 174 174 174 292 292 292 F F F 10 200-219 172 172 172 285 285 285 F F F 11 220-239 172 172 172 285 285 285 T T T 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 174 174 174 290 290 290 F F F 16 320-339 174 174 174 290 290 290 F F F 17 340-359 172 172 172 288 288 288 F F F 18 360-379 174 174 174 291 291 291 F F F 19 380-399 174 174 174 291 291 291 F F F 20 400-419 174 174 174 290 290 290 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 170 170 170 283 283 283 T T T 23 460-479 170 170 170 283 283 283 T T T 24 480-499 173 173 173 289 289 289 F F F 25 500-519 169 169 169 281 281 281 F F F 26 520-539 174 174 174 291 291 291 T T T 27 540-559 174 174 174 291 291 291 T T T 28 560-579 174 174 174 290 290 290 F F F 29 580-599 170 170 170 282 282 282 F F F Max gen: 174, Max flush: 292 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 286 286 286 F F F 1 020-039 170 170 170 282 282 282 F F F 2 040-059 172 172 172 285 285 285 F F F 3 060-079 174 174 174 290 290 290 F F F 4 080-099 172 172 172 286 286 286 F F F 5 100-119 174 174 174 290 290 290 F F F 6 120-139 174 174 174 290 290 290 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 174 174 174 292 292 292 F F F 10 200-219 172 172 172 285 285 285 F F F 11 220-239 175 175 175 293 293 293 F F F 12 240-259 165 165 165 277 277 277 F F F 13 260-279 165 165 165 277 277 277 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 174 174 174 290 290 290 F F F 16 320-339 174 174 174 290 290 290 F F F 17 340-359 172 172 172 288 288 288 F F F 18 360-379 174 174 174 291 291 291 F F F 19 380-399 174 174 174 291 291 291 F F F 20 400-419 174 174 174 290 290 290 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 175 175 175 293 293 293 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 169 169 169 281 281 281 F F F 26 520-539 175 175 175 293 293 293 F F F 27 540-559 175 175 175 293 293 293 F F F 28 560-579 174 174 174 290 290 290 F F F 29 580-599 170 170 170 282 282 282 F F F Max gen: 175, Max flush: 293 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.366175029Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:41.4085272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.408920818Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.43994006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.440561905Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.440596231Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.440610569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.440624798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896} {"msg":"Crucible 7b2898a6-2a35-47b9-84ba-2b6fbf79c37b has session id: d6a91619-3dcd-41a3-bc4b-176873690d05","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441210116Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Upstairs opts: Upstairs UUID: 7b2898a6-2a35-47b9-84ba-2b6fbf79c37b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441238343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Crucible stats registered with UUID: 7b2898a6-2a35-47b9-84ba-2b6fbf79c37b","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441252792Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:41.441265091Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441694765Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441718523Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896} {"msg":"7b2898a6-2a35-47b9-84ba-2b6fbf79c37b active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441785926Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441805014Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"0","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.44183972Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"0","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441854449Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"1","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441868667Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"1","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441881656Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"2","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441895414Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"2","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441908093Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.441921312Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.44193445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"ds_connection connected from Ok(127.0.0.1:52517)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.442112711Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"ds_connection connected from Ok(127.0.0.1:62287)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.442170855Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"ds_connection connected from Ok(127.0.0.1:65206)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.442204352Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.442945554Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"2","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.44298458Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"1","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.443154092Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"0","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.443631052Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"1","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.443659359Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"2","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.443674977Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"0","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444238848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444340037Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[0]R flush_numbers[0..12]: [286, 282, 285, 290, 286, 290, 290, 290, 284, 292, 285, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444402501Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[0]R generation[0..12]: [172, 170, 172, 174, 172, 174, 174, 174, 171, 174, 172, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444421659Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444435838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[1]R flush_numbers[0..12]: [286, 282, 285, 290, 286, 290, 290, 290, 284, 292, 285, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444449446Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[1]R generation[0..12]: [172, 170, 172, 174, 172, 174, 174, 174, 171, 174, 172, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444462935Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444476463Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[2]R flush_numbers[0..12]: [286, 282, 285, 290, 286, 290, 290, 290, 284, 292, 285, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444489672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[2]R generation[0..12]: [172, 170, 172, 174, 172, 174, 174, 174, 171, 174, 172, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.44450286Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444516349Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444533307Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444546716Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444559245Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444571533Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"":"downstairs","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444584222Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":Wait for a query_work_queue command to finish before sending IO "Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.44460238Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"0","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444615288Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"1","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444627807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"client":"2","":"downstairs","session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444640546Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444652425Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"7b2898a6-2a35-47b9-84ba-2b6fbf79c37b is now active with session: d6a91619-3dcd-41a3-bc4b-176873690d05","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444664953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444676492Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896,"session_id":"d6a91619-3dcd-41a3-bc4b-176873690d05"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.444688331Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896} {"msg":"Activated sub_volume 7b2898a6-2a35-47b9-84ba-2b6fbf79c37b","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:41.44469912Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 592 len 20480 data: 13 15 13 12 11 03/30 Read block 381 len 40960 04/30 Flush 05/30 Write block 347 len 8192 data: 10 10 06/30 Write block 247 len 12288 data: 14 14 13 07/30 Write block 345 len 12288 data: 11 10 11 08/30 Write block 262 len 32768 data: 11 10 9 12 15 15 20 18 09/30 Read block 289 len 20480 10/30 Read block 582 len 40960 11/30 Flush 12/30 Flush 13/30 Write block 3 len 36864 data: 9 10 11 15 15 15 13 13 12 14/30 Read block 426 len 20480 15/30 Read block 43 len 20480 16/30 Read block 493 len 16384 17/30 Read block 4 len 20480 18/30 Write block 356 len 40960 data: 10 10 8 9 9 8 8 9 8 11 19/30 Read block 160 len 16384 20/30 Read block 243 len 16384 21/30 Write block 531 len 36864 data: 11 13 13 16 16 15 14 15 15 22/30 Write block 586 len 4096 data: 16 23/30 Read block 481 len 24576 24/30 Write block 519 len 20480 data: 18 17 17 15 12 25/30 Read block 317 len 24576 26/30 Flush 27/30 Write block 267 len 12288 data: 16 21 19 28/30 Read block 479 len 36864 29/30 Write block 396 len 4096 data: 18 30/30 Read block 192 len 4096 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:41.465531701Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1896} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 298 298 298 F F F 1 020-039 170 170 170 282 282 282 F F F 2 040-059 172 172 172 285 285 285 F F F 3 060-079 174 174 174 290 290 290 F F F 4 080-099 172 172 172 286 286 286 F F F 5 100-119 174 174 174 290 290 290 F F F 6 120-139 174 174 174 290 290 290 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 174 174 174 292 292 292 F F F 10 200-219 172 172 172 285 285 285 F F F 11 220-239 175 175 175 293 293 293 F F F 12 240-259 176 176 176 296 296 296 F F F 13 260-279 176 176 176 296 296 296 T T T 14 280-299 169 169 169 281 281 281 F F F 15 300-319 174 174 174 290 290 290 F F F 16 320-339 174 174 174 290 290 290 F F F 17 340-359 176 176 176 298 298 298 F F F 18 360-379 176 176 176 298 298 298 F F F 19 380-399 174 174 174 291 291 291 T T T 20 400-419 174 174 174 290 290 290 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 175 175 175 293 293 293 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 176 176 176 298 298 298 F F F 26 520-539 176 176 176 298 298 298 F F F 27 540-559 175 175 175 293 293 293 F F F 28 560-579 174 174 174 290 290 290 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 176, Max flush: 298 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 298 298 298 F F F 1 020-039 170 170 170 282 282 282 F F F 2 040-059 172 172 172 285 285 285 F F F 3 060-079 174 174 174 290 290 290 F F F 4 080-099 172 172 172 286 286 286 F F F 5 100-119 174 174 174 290 290 290 F F F 6 120-139 174 174 174 290 290 290 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 174 174 174 292 292 292 F F F 10 200-219 172 172 172 285 285 285 F F F 11 220-239 175 175 175 293 293 293 F F F 12 240-259 176 176 176 296 296 296 F F F 13 260-279 177 177 177 299 299 299 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 174 174 174 290 290 290 F F F 16 320-339 174 174 174 290 290 290 F F F 17 340-359 176 176 176 298 298 298 F F F 18 360-379 176 176 176 298 298 298 F F F 19 380-399 177 177 177 299 299 299 F F F 20 400-419 174 174 174 290 290 290 F F F 21 420-439 162 162 162 273 273 273 F F F 22 440-459 175 175 175 293 293 293 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 176 176 176 298 298 298 F F F 26 520-539 176 176 176 298 298 298 F F F 27 540-559 175 175 175 293 293 293 F F F 28 560-579 174 174 174 290 290 290 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 177, Max flush: 299 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.720110698Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.765072744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.765653953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.76568728Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.765701178Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.765715357Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904} {"msg":"Crucible e42a849b-7368-4ac3-8b5e-8ec1cd6c9284 has session id: 3070451f-0e77-432f-ab97-95d7a7a9b218","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.766231973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Upstairs opts: Upstairs UUID: e42a849b-7368-4ac3-8b5e-8ec1cd6c9284, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.766271398Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Crucible stats registered with UUID: e42a849b-7368-4ac3-8b5e-8ec1cd6c9284","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.766299775Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:45.766318573Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.76682666Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.766875215Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904} {"msg":"e42a849b-7368-4ac3-8b5e-8ec1cd6c9284 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.766938508Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.766955237Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"0","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.766980644Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"0","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767009291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767025539Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767044387Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"2","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767067265Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"2","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767090163Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.76711295Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767136858Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"ds_connection connected from Ok(127.0.0.1:47817)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767160415Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"ds_connection connected from Ok(127.0.0.1:43714)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767184982Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:45.767218349Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:45.767244946Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:45.767321398Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767353225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767857872Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"2","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.767884569Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"0","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.7685423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"0","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.768569457Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"2","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.769175413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:45.769282072Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.768414615Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"ds_connection connected from Ok(127.0.0.1:64815)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.768608575Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.769579343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.77017624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[0]R flush_numbers[0..12]: [298, 282, 285, 290, 286, 290, 290, 290, 284, 292, 285, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770777357Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[0]R generation[0..12]: [176, 170, 172, 174, 172, 174, 174, 174, 171, 174, 172, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770808304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770828671Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[1]R flush_numbers[0..12]: [298, 282, 285, 290, 286, 290, 290, 290, 284, 292, 285, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.77084591Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[1]R generation[0..12]: [176, 170, 172, 174, 172, 174, 174, 174, 171, 174, 172, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770863088Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770880176Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[2]R flush_numbers[0..12]: [298, 282, 285, 290, 286, 290, 290, 290, 284, 292, 285, 293]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770897014Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[2]R generation[0..12]: [176, 170, 172, 174, 172, 174, 174, 174, 171, 174, 172, 175]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770913953Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770930811Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770946789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770963427Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.770996584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771013112Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"":"downstairs","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.77102891Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771054928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"0","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771071576Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"1","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771087624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"client":"2","":"downstairs","session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771103633Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771118361Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"e42a849b-7368-4ac3-8b5e-8ec1cd6c9284 is now active with session: 3070451f-0e77-432f-ab97-95d7a7a9b218","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.77113391Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771148348Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904,"session_id":"3070451f-0e77-432f-ab97-95d7a7a9b218"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771163246Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904} {"msg":"Activated sub_volume e42a849b-7368-4ac3-8b5e-8ec1cd6c9284","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:28:55.771176735Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 36864 data: 15 15 16 15 16 15 16 13 11 02/30 Read block 216 len 36864 03/30 Read block 305 len 24576 04/30 Read block 52 len 40960 05/30 Flush 06/30 Write block 79 len 36864 data: 12 14 14 13 10 12 11 10 10 07/30 Write block 416 len 24576 data: 18 18 15 16 17 12 08/30 Read block 145 len 32768 09/30 Read block 400 len 28672 10/30 Read block 492 len 16384 11/30 Read block 330 len 40960 12/30 Read block 139 len 32768 13/30 Write block 392 len 32768 data: 18 17 16 17 19 16 15 17 14/30 Write block 270 len 12288 data: 21 22 20 15/30 Read block 582 len 12288 16/30 Write block 543 len 32768 data: 18 14 12 13 12 12 13 13 17/30 Read block 15 len 40960 18/30 Write block 393 len 40960 data: 18 17 18 20 17 16 18 17 17 19 19/30 Write block 55 len 4096 data: 19 20/30 Write block 443 len 24576 data: 10 11 10 13 14 13 21/30 Flush 22/30 Write block 91 len 36864 data: 8 9 9 6 8 13 13 13 18 23/30 Write block 23 len 12288 data: 10 9 8 24/30 Read block 65 len 40960 25/30 Read block 375 len 20480 26/30 Write block 74 len 16384 data: 11 10 8 14 27/30 Read block 189 len 32768 28/30 Write block 566 len 16384 data: 19 17 14 13 29/30 Write block 195 len 24576 data: 15 18 16 12 12 14 30/30 Write block 228 len 36864 data: 8 9 8 10 11 10 12 13 13 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:28:55.793715277Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1904} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 298 298 298 F F F 1 020-039 178 178 178 300 300 300 T T T 2 040-059 178 178 178 301 301 301 F F F 3 060-079 178 178 178 301 301 301 T T T 4 080-099 178 178 178 301 301 301 T T T 5 100-119 174 174 174 290 290 290 F F F 6 120-139 174 174 174 290 290 290 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 174 174 174 292 292 292 T T T 10 200-219 172 172 172 285 285 285 T T T 11 220-239 175 175 175 293 293 293 T T T 12 240-259 176 176 176 296 296 296 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 174 174 174 290 290 290 F F F 16 320-339 174 174 174 290 290 290 F F F 17 340-359 176 176 176 298 298 298 F F F 18 360-379 176 176 176 298 298 298 F F F 19 380-399 178 178 178 301 301 301 F F F 20 400-419 178 178 178 301 301 301 F F F 21 420-439 178 178 178 301 301 301 F F F 22 440-459 178 178 178 301 301 301 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 176 176 176 298 298 298 F F F 26 520-539 176 176 176 298 298 298 F F F 27 540-559 178 178 178 301 301 301 F F F 28 560-579 174 174 174 290 290 290 T T T 29 580-599 176 176 176 298 298 298 F F F Max gen: 178, Max flush: 301 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 298 298 298 F F F 1 020-039 179 179 179 302 302 302 F F F 2 040-059 178 178 178 301 301 301 F F F 3 060-079 179 179 179 302 302 302 F F F 4 080-099 179 179 179 302 302 302 F F F 5 100-119 174 174 174 290 290 290 F F F 6 120-139 174 174 174 290 290 290 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 179 179 179 302 302 302 F F F 10 200-219 179 179 179 302 302 302 F F F 11 220-239 179 179 179 302 302 302 F F F 12 240-259 176 176 176 296 296 296 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 174 174 174 290 290 290 F F F 16 320-339 174 174 174 290 290 290 F F F 17 340-359 176 176 176 298 298 298 F F F 18 360-379 176 176 176 298 298 298 F F F 19 380-399 178 178 178 301 301 301 F F F 20 400-419 178 178 178 301 301 301 F F F 21 420-439 178 178 178 301 301 301 F F F 22 440-459 178 178 178 301 301 301 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 176 176 176 298 298 298 F F F 26 520-539 176 176 176 298 298 298 F F F 27 540-559 178 178 178 301 301 301 F F F 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 179, Max flush: 302 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.04908176Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.092734525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.093384996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.093419903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.093434071Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.09344828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912} {"msg":"Crucible b1ddfc14-d931-4e20-be13-56e5d557de23 has session id: 70dabd38-ca2d-4229-bea6-ffa3b4de3010","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094063335Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Upstairs opts: Upstairs UUID: b1ddfc14-d931-4e20-be13-56e5d557de23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094103441Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Crucible stats registered with UUID: b1ddfc14-d931-4e20-be13-56e5d557de23","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094127128Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:00.094147166Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094654263Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094692719Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912} {"msg":"b1ddfc14-d931-4e20-be13-56e5d557de23 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094759182Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.09477648Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"0","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094803807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"0","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094817926Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094832344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094852722Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"2","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094867481Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"2","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094889148Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094908236Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.094922215Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:00.095009606Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:00.095049222Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:00.095084328Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"ds_connection connected from Ok(127.0.0.1:38302)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.095116004Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"ds_connection connected from Ok(127.0.0.1:44354)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.095143372Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.095290026Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.095792313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"0","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.095828789Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"2","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.096429757Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"2","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.096457294Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"0","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.096980539Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:00.097109265Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.09576453Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"ds_connection connected from Ok(127.0.0.1:63266)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.095845092Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.097046675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.097626525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[0]R flush_numbers[0..12]: [298, 302, 301, 302, 302, 290, 290, 290, 284, 302, 302, 302]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098212883Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[0]R generation[0..12]: [176, 179, 178, 179, 179, 174, 174, 174, 171, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.09823812Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098258678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[1]R flush_numbers[0..12]: [298, 302, 301, 302, 302, 290, 290, 290, 284, 302, 302, 302]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098273657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[1]R generation[0..12]: [176, 179, 178, 179, 179, 174, 174, 174, 171, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098287195Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098300884Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[2]R flush_numbers[0..12]: [298, 302, 301, 302, 302, 290, 290, 290, 284, 302, 302, 302]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098314412Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[2]R generation[0..12]: [176, 179, 178, 179, 179, 174, 174, 174, 171, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098327681Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098341059Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098353538Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098366397Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098378975Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098391394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"":"downstairs","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098403923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098415642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"0","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:29:10.09842864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"1","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098448928Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"client":"2","":"downstairs","session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098463007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098475225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"b1ddfc14-d931-4e20-be13-56e5d557de23 is now active with session: 70dabd38-ca2d-4229-bea6-ffa3b4de3010","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098487734Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098499373Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912,"session_id":"70dabd38-ca2d-4229-bea6-ffa3b4de3010"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.098511581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912} {"msg":"Activated sub_volume b1ddfc14-d931-4e20-be13-56e5d557de23","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:10.09852291Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 511 len 40960 data: 19 16 18 16 16 16 19 19 19 18 02/30 Write block 317 len 24576 data: 13 11 13 11 11 10 03/30 Write block 364 len 20480 data: 9 12 12 13 13 04/30 Read block 397 len 24576 05/30 Flush 06/30 Flush 07/30 Write block 214 len 36864 data: 18 19 21 23 21 22 21 19 16 08/30 Flush 09/30 Write block 557 len 8192 data: 16 20 10/30 Flush 11/30 Write block 21 len 28672 data: 11 10 11 10 9 9 8 12/30 Read block 25 len 20480 13/30 Flush 14/30 Write block 11 len 12288 data: 13 13 11 15/30 Flush 16/30 Write block 539 len 32768 data: 16 13 14 15 19 15 13 14 17/30 Read block 17 len 20480 18/30 Write block 118 len 24576 data: 10 12 13 16 15 16 19/30 Write block 138 len 4096 data: 17 20/30 Read block 170 len 36864 21/30 Read block 194 len 24576 22/30 Read block 130 len 8192 23/30 Read block 244 len 24576 24/30 Read block 299 len 28672 25/30 Read block 262 len 16384 26/30 Read block 254 len 4096 27/30 Read block 531 len 12288 28/30 Read block 449 len 24576 29/30 Write block 225 len 16384 data: 14 11 10 9 30/30 Read block 374 len 4096 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 15 14 13 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:10.118936766Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1912} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 308 308 308 F F F 1 020-039 180 180 180 307 307 307 F F F 2 040-059 178 178 178 301 301 301 F F F 3 060-079 179 179 179 302 302 302 F F F 4 080-099 179 179 179 302 302 302 F F F 5 100-119 174 174 174 290 290 290 T T T 6 120-139 174 174 174 290 290 290 T T T 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 179 179 179 302 302 302 F F F 10 200-219 180 180 180 305 305 305 F F F 11 220-239 180 180 180 305 305 305 T T T 12 240-259 176 176 176 296 296 296 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 180 180 180 303 303 303 F F F 16 320-339 180 180 180 303 303 303 F F F 17 340-359 176 176 176 298 298 298 F F F 18 360-379 180 180 180 303 303 303 F F F 19 380-399 178 178 178 301 301 301 F F F 20 400-419 178 178 178 301 301 301 F F F 21 420-439 178 178 178 301 301 301 F F F 22 440-459 178 178 178 301 301 301 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 180 180 180 303 303 303 F F F 26 520-539 180 180 180 303 303 303 T T T 27 540-559 180 180 180 306 306 306 T T T 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 180, Max flush: 308 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 308 308 308 F F F 1 020-039 180 180 180 307 307 307 F F F 2 040-059 178 178 178 301 301 301 F F F 3 060-079 179 179 179 302 302 302 F F F 4 080-099 179 179 179 302 302 302 F F F 5 100-119 181 181 181 309 309 309 F F F 6 120-139 181 181 181 309 309 309 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 179 179 179 302 302 302 F F F 10 200-219 180 180 180 305 305 305 F F F 11 220-239 181 181 181 309 309 309 F F F 12 240-259 176 176 176 296 296 296 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 180 180 180 303 303 303 F F F 16 320-339 180 180 180 303 303 303 F F F 17 340-359 176 176 176 298 298 298 F F F 18 360-379 180 180 180 303 303 303 F F F 19 380-399 178 178 178 301 301 301 F F F 20 400-419 178 178 178 301 301 301 F F F 21 420-439 178 178 178 301 301 301 F F F 22 440-459 178 178 178 301 301 301 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 180 180 180 303 303 303 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 181 181 181 309 309 309 F F F 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 181, Max flush: 309 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.363205795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:14.405591122Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.405973802Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.436687466Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.437230558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.437265185Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.437278713Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.437292632Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920} {"msg":"Crucible 8dcb8743-1e9d-45c9-98be-de5144f55f07 has session id: f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.437911937Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Upstairs opts: Upstairs UUID: 8dcb8743-1e9d-45c9-98be-de5144f55f07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.437948423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Crucible stats registered with UUID: 8dcb8743-1e9d-45c9-98be-de5144f55f07","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.437965661Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:14.43797943Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438496605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438527842Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920} {"msg":"8dcb8743-1e9d-45c9-98be-de5144f55f07 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438587336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438613933Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"0","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.43864313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"0","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438658339Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"1","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438673047Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"1","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438686756Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"2","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438701564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"2","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438715662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.43873882Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438762777Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"ds_connection connected from Ok(127.0.0.1:48073)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438804343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"ds_connection connected from Ok(127.0.0.1:57252)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.438846149Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"ds_connection connected from Ok(127.0.0.1:50959)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.439032939Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.43959897Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"2","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.439627467Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"1","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.439651774Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"0","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.440238003Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"2","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.440316784Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"1","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.440427283Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"0","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.440823121Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.440968076Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[0]R flush_numbers[0..12]: [308, 307, 301, 302, 302, 309, 309, 290, 284, 302, 305, 309]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441032799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[0]R generation[0..12]: [180, 180, 178, 179, 179, 181, 181, 174, 171, 179, 180, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441052217Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441067706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[1]R flush_numbers[0..12]: [308, 307, 301, 302, 302, 309, 309, 290, 284, 302, 305, 309]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441081834Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[1]R generation[0..12]: [180, 180, 178, 179, 179, 181, 181, 174, 171, 179, 180, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441095343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441109171Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[2]R flush_numbers[0..12]: [308, 307, 301, 302, 302, 309, 309, 290, 284, 302, 305, 309]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.44112287Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[2]R generation[0..12]: [180, 180, 178, 179, 179, 181, 181, 174, 171, 179, 180, 181]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441139908Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441153826Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441166385Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441179214Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441191733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441204241Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"":"downstairs","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.44121695Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441228598Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"0","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441248007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"1","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441260815Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"client":"2","":"downstairs","session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441273514Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441285332Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"8dcb8743-1e9d-45c9-98be-de5144f55f07 is now active with session: f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441297821Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.44130947Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920,"session_id":"f5fb98a3-b6a2-4729-9ae9-c7c35c0d8173"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441321639Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920} {"msg":"Activated sub_volume 8dcb8743-1e9d-45c9-98be-de5144f55f07","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:14.441332498Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 94 len 40960 data: 7 9 14 14 14 19 20 17 20 19 02/30 Flush 03/30 Write block 413 len 36864 data: 15 16 16 19 19 16 17 18 13 04/30 Read block 215 len 36864 05/30 Read block 543 len 32768 06/30 Read block 13 len 8192 07/30 Write block 25 len 12288 data: 10 10 9 08/30 Write block 108 len 8192 data: 15 12 09/30 Write block 517 len 8192 data: 20 20 10/30 Read block 527 len 32768 11/30 Write block 430 len 20480 data: 23 23 21 17 18 12/30 Write block 335 len 32768 data: 20 18 19 21 20 18 20 16 13/30 Flush 14/30 Write block 257 len 12288 data: 9 10 7 15/30 Flush 16/30 Write block 544 len 12288 data: 16 14 15 17/30 Read block 63 len 20480 18/30 Write block 545 len 24576 data: 15 16 13 13 14 14 19/30 Write block 230 len 36864 data: 9 11 12 11 13 14 14 13 13 20/30 Write block 373 len 40960 data: 14 12 10 12 11 14 14 13 11 9 21/30 Write block 67 len 12288 data: 16 15 17 22/30 Read block 471 len 20480 23/30 Write block 198 len 20480 data: 13 13 15 13 11 24/30 Write block 22 len 16384 data: 11 12 11 11 25/30 Read block 541 len 36864 26/30 Read block 212 len 36864 27/30 Write block 253 len 12288 data: 12 12 13 28/30 Read block 337 len 40960 29/30 Read block 201 len 4096 30/30 Write block 69 len 8192 data: 18 15 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 3 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:14.461457344Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1920} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 308 308 308 F F F 1 020-039 182 182 182 311 311 311 T T T 2 040-059 178 178 178 301 301 301 F F F 3 060-079 179 179 179 302 302 302 T T T 4 080-099 182 182 182 310 310 310 F F F 5 100-119 182 182 182 311 311 311 F F F 6 120-139 181 181 181 309 309 309 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 179 179 179 302 302 302 T T T 10 200-219 180 180 180 305 305 305 T T T 11 220-239 181 181 181 309 309 309 T T T 12 240-259 182 182 182 312 312 312 T T T 13 260-279 178 178 178 301 301 301 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 180 180 180 303 303 303 F F F 16 320-339 182 182 182 311 311 311 F F F 17 340-359 182 182 182 311 311 311 F F F 18 360-379 180 180 180 303 303 303 T T T 19 380-399 178 178 178 301 301 301 T T T 20 400-419 182 182 182 311 311 311 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 178 178 178 301 301 301 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 181 181 181 309 309 309 T T T 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 182, Max flush: 312 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 308 308 308 F F F 1 020-039 183 183 183 313 313 313 F F F 2 040-059 178 178 178 301 301 301 F F F 3 060-079 183 183 183 313 313 313 F F F 4 080-099 182 182 182 310 310 310 F F F 5 100-119 182 182 182 311 311 311 F F F 6 120-139 181 181 181 309 309 309 F F F 7 140-159 174 174 174 290 290 290 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 183 183 183 313 313 313 F F F 10 200-219 183 183 183 313 313 313 F F F 11 220-239 183 183 183 313 313 313 F F F 12 240-259 183 183 183 313 313 313 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 180 180 180 303 303 303 F F F 16 320-339 182 182 182 311 311 311 F F F 17 340-359 182 182 182 311 311 311 F F F 18 360-379 183 183 183 313 313 313 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 182 182 182 311 311 311 F F F 21 420-439 182 182 182 311 311 311 F F F 22 440-459 178 178 178 301 301 301 F F F 23 460-479 175 175 175 293 293 293 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 183 183 183 313 313 313 F F F 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 183, Max flush: 313 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.750540496Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.794599417Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.795203643Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.795245819Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.795268536Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.795288054Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928} {"msg":"Crucible 1f5f1446-1fde-42b8-b503-02d1eb13d967 has session id: ddaf437a-3337-4d97-a09c-2dd14d256fd5","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.795880852Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Upstairs opts: Upstairs UUID: 1f5f1446-1fde-42b8-b503-02d1eb13d967, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.795919518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Crucible stats registered with UUID: 1f5f1446-1fde-42b8-b503-02d1eb13d967","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.795953924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:18.795970363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796435264Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.79646921Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928} {"msg":"1f5f1446-1fde-42b8-b503-02d1eb13d967 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796522325Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796548422Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"0","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796577769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"0","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796592157Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"1","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796606236Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"1","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796619104Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"2","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796632693Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"2","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796649691Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.79666429Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796678308Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51602)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796866309Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36070)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796903474Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:62952)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.796932591Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.797501732Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"0","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.797540907Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"2","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.798166552Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"0","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.798210187Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"2","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.798456831Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"1","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.798874878Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.798910484Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799006324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"1","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[0]R flush_numbers[0..12]: [308, 313, 301, 313, 310, 311, 309, 290, 284, 313, 313, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799502301Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[0]R generation[0..12]: [180, 183, 178, 183, 182, 182, 181, 174, 171, 183, 183, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.79952229Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799536838Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[1]R flush_numbers[0..12]: [308, 313, 301, 313, 310, 311, 309, 290, 284, 313, 313, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799550236Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[1]R generation[0..12]: [180, 183, 178, 183, 182, 182, 181, 174, 171, 183, 183, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799563465Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799576904Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[2]R flush_numbers[0..12]: [308, 313, 301, 313, 310, 311, 309, 290, 284, 313, 313, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799590092Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[2]R generation[0..12]: [180, 183, 178, 183, 182, 182, 181, 174, 171, 183, 183, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799603531Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799618139Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799630808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799653116Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799684233Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.79970793Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"":"downstairs","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799725418Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799750206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"0","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799774743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"1","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.79979924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"client":"2","":"downstairs","session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799822438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799842396Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"1f5f1446-1fde-42b8-b503-02d1eb13d967 is now active with session: ddaf437a-3337-4d97-a09c-2dd14d256fd5","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:29:18.799865243Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799889751Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928,"session_id":"ddaf437a-3337-4d97-a09c-2dd14d256fd5"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799906879Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928} {"msg":"Activated sub_volume 1f5f1446-1fde-42b8-b503-02d1eb13d967","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:18.799918008Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 208 len 36864 data: 12 13 14 15 15 17 19 20 22 02/30 Read block 241 len 40960 03/30 Write block 115 len 8192 data: 9 8 04/30 Read block 212 len 8192 05/30 Write block 466 len 4096 data: 18 06/30 Read block 28 len 12288 07/30 Read block 200 len 16384 08/30 Read block 42 len 20480 09/30 Flush 10/30 Write block 29 len 36864 data: 13 16 16 16 17 16 17 16 17 11/30 Read block 580 len 40960 12/30 Write block 556 len 12288 data: 15 17 21 13/30 Write block 204 len 12288 data: 9 13 12 14/30 Read block 39 len 32768 15/30 Read block 73 len 8192 16/30 Read block 499 len 16384 17/30 Write block 550 len 32768 data: 15 14 15 16 13 16 16 18 18/30 Read block 227 len 12288 19/30 Write block 408 len 28672 data: 17 15 15 16 14 16 17 20/30 Read block 240 len 20480 21/30 Write block 549 len 40960 data: 15 16 15 16 17 14 17 17 19 22 22/30 Read block 275 len 12288 23/30 Write block 435 len 40960 data: 17 15 13 11 8 11 10 9 11 12 24/30 Read block 206 len 36864 25/30 Read block 412 len 4096 26/30 Flush 27/30 Write block 463 len 40960 data: 18 17 15 19 16 17 18 21 21 17 28/30 Write block 143 len 4096 data: 19 29/30 Write block 9 len 40960 data: 14 14 14 14 12 13 14 13 14 11 30/30 Write block 105 len 8192 data: 14 14 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:18.82065078Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1928} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 308 308 308 T T T 1 020-039 184 184 184 315 315 315 F F F 2 040-059 178 178 178 301 301 301 F F F 3 060-079 183 183 183 313 313 313 F F F 4 080-099 182 182 182 310 310 310 F F F 5 100-119 184 184 184 314 314 314 T T T 6 120-139 181 181 181 309 309 309 F F F 7 140-159 174 174 174 290 290 290 T T T 8 160-179 171 171 171 284 284 284 F F F 9 180-199 183 183 183 313 313 313 F F F 10 200-219 184 184 184 315 315 315 F F F 11 220-239 183 183 183 313 313 313 F F F 12 240-259 183 183 183 313 313 313 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 180 180 180 303 303 303 F F F 16 320-339 182 182 182 311 311 311 F F F 17 340-359 182 182 182 311 311 311 F F F 18 360-379 183 183 183 313 313 313 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 184 184 184 315 315 315 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 184 184 184 315 315 315 F F F 23 460-479 184 184 184 314 314 314 T T T 24 480-499 173 173 173 289 289 289 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 184 184 184 315 315 315 F F F 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 184, Max flush: 315 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 185 185 185 316 316 316 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 178 178 178 301 301 301 F F F 3 060-079 183 183 183 313 313 313 F F F 4 080-099 182 182 182 310 310 310 F F F 5 100-119 185 185 185 316 316 316 F F F 6 120-139 181 181 181 309 309 309 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 183 183 183 313 313 313 F F F 10 200-219 184 184 184 315 315 315 F F F 11 220-239 183 183 183 313 313 313 F F F 12 240-259 183 183 183 313 313 313 F F F 13 260-279 178 178 178 301 301 301 F F F 14 280-299 169 169 169 281 281 281 F F F 15 300-319 180 180 180 303 303 303 F F F 16 320-339 182 182 182 311 311 311 F F F 17 340-359 182 182 182 311 311 311 F F F 18 360-379 183 183 183 313 313 313 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 184 184 184 315 315 315 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 184 184 184 315 315 315 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 173 173 173 289 289 289 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 184 184 184 315 315 315 F F F 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 185, Max flush: 316 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.062984733Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.106836756Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.107406766Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.107441852Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.107455911Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.107469989Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936} {"msg":"Crucible 81344907-4748-495b-a401-e7abf6ddcece has session id: ede13098-ec46-4281-b66f-7a98b655d8fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.107992605Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Upstairs opts: Upstairs UUID: 81344907-4748-495b-a401-e7abf6ddcece, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108025901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Crucible stats registered with UUID: 81344907-4748-495b-a401-e7abf6ddcece","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108063887Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:23.108078736Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108496452Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108528638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936} {"msg":"81344907-4748-495b-a401-e7abf6ddcece active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108582672Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108615109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"0","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108660304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"0","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108688692Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"1","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108717118Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"1","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108742476Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108768813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.10879528Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108814808Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108828507Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:23.108853324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:23.108883341Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:23.108909828Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:37586)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108930936Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:49987)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108955404Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.108980951Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.109512375Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"0","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.109538432Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"1","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.110139379Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"1","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.110273145Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"0","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.110664584Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:23.110773462Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.110761408Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:38169)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.110919701Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.112339862Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.11283814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[0]R flush_numbers[0..12]: [316, 315, 301, 313, 310, 316, 309, 316, 284, 313, 315, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.11340952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[0]R generation[0..12]: [185, 184, 178, 183, 182, 185, 181, 185, 171, 183, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113434837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113449996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[1]R flush_numbers[0..12]: [316, 315, 301, 313, 310, 316, 309, 316, 284, 313, 315, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113463444Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[1]R generation[0..12]: [185, 184, 178, 183, 182, 185, 181, 185, 171, 183, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113483002Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113497201Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[2]R flush_numbers[0..12]: [316, 315, 301, 313, 310, 316, 309, 316, 284, 313, 315, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113511049Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[2]R generation[0..12]: [185, 184, 178, 183, 182, 185, 181, 185, 171, 183, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113524698Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113538296Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113550915Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113563714Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113576352Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113588791Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"":"downstairs","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.11360149Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113613058Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"0","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113626137Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"1","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113638846Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"client":"2","":"downstairs","session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} Wait for a query_work_queue command to finish before sending IO {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113659924Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113674382Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"81344907-4748-495b-a401-e7abf6ddcece is now active with session: ede13098-ec46-4281-b66f-7a98b655d8fd","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113687131Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113698629Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936,"session_id":"ede13098-ec46-4281-b66f-7a98b655d8fd"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113710358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936} {"msg":"Activated sub_volume 81344907-4748-495b-a401-e7abf6ddcece","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:33.113721257Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 259 len 4096 data: 8 02/30 Read block 154 len 16384 03/30 Read block 305 len 32768 04/30 Flush 05/30 Write block 277 len 16384 data: 10 13 17 16 06/30 Flush 07/30 Flush 08/30 Write block 408 len 4096 data: 18 09/30 Read block 225 len 8192 10/30 Write block 43 len 36864 data: 16 16 15 15 17 17 18 19 17 11/30 Read block 544 len 32768 12/30 Read block 486 len 28672 13/30 Read block 584 len 28672 14/30 Read block 133 len 32768 15/30 Read block 230 len 40960 16/30 Read block 83 len 16384 17/30 Read block 408 len 12288 18/30 Read block 171 len 28672 19/30 Read block 128 len 4096 20/30 Write block 9 len 24576 data: 15 15 15 15 13 14 21/30 Read block 87 len 8192 22/30 Write block 444 len 12288 data: 13 11 14 23/30 Read block 75 len 32768 24/30 Read block 186 len 20480 25/30 Write block 481 len 32768 data: 15 14 13 15 16 17 18 16 26/30 Flush 27/30 Write block 280 len 8192 data: 17 17 28/30 Read block 363 len 16384 29/30 Read block 31 len 16384 30/30 Read block 30 len 8192 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 4 3 4 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:33.134620582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1936} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 320 320 320 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 186 186 186 320 320 320 F F F 3 060-079 183 183 183 313 313 313 F F F 4 080-099 182 182 182 310 310 310 F F F 5 100-119 185 185 185 316 316 316 F F F 6 120-139 181 181 181 309 309 309 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 183 183 183 313 313 313 F F F 10 200-219 184 184 184 315 315 315 F F F 11 220-239 183 183 183 313 313 313 F F F 12 240-259 186 186 186 317 317 317 F F F 13 260-279 186 186 186 318 318 318 F F F 14 280-299 186 186 186 318 318 318 T T T 15 300-319 180 180 180 303 303 303 F F F 16 320-339 182 182 182 311 311 311 F F F 17 340-359 182 182 182 311 311 311 F F F 18 360-379 183 183 183 313 313 313 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 186 186 186 320 320 320 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 186 186 186 320 320 320 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 186 186 186 320 320 320 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 184 184 184 315 315 315 F F F 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 186, Max flush: 320 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 320 320 320 F F F 1 020-039 184 184 184 315 315 315 F F F 2 040-059 186 186 186 320 320 320 F F F 3 060-079 183 183 183 313 313 313 F F F 4 080-099 182 182 182 310 310 310 F F F 5 100-119 185 185 185 316 316 316 F F F 6 120-139 181 181 181 309 309 309 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 183 183 183 313 313 313 F F F 10 200-219 184 184 184 315 315 315 F F F 11 220-239 183 183 183 313 313 313 F F F 12 240-259 186 186 186 317 317 317 F F F 13 260-279 186 186 186 318 318 318 F F F 14 280-299 187 187 187 321 321 321 F F F 15 300-319 180 180 180 303 303 303 F F F 16 320-339 182 182 182 311 311 311 F F F 17 340-359 182 182 182 311 311 311 F F F 18 360-379 183 183 183 313 313 313 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 186 186 186 320 320 320 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 186 186 186 320 320 320 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 186 186 186 320 320 320 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 184 184 184 315 315 315 F F F 28 560-579 179 179 179 302 302 302 F F F 29 580-599 176 176 176 298 298 298 F F F Max gen: 187, Max flush: 321 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.353999706Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.397572139Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.398169536Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.398214561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.398238298Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.398257136Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944} {"msg":"Crucible c2a12323-f7df-43fc-b182-296f260c3008 has session id: a82ff958-ea8c-429e-95bd-c5e4a4ad712f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.398778972Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Upstairs opts: Upstairs UUID: c2a12323-f7df-43fc-b182-296f260c3008, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.398819468Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Crucible stats registered with UUID: c2a12323-f7df-43fc-b182-296f260c3008","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.398837376Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:37.398851744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399220235Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399245563Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944} {"msg":"c2a12323-f7df-43fc-b182-296f260c3008 active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399305277Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399323575Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"0","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399356581Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"0","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399380898Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399408046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399435323Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"2","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.3994662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"2","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399489687Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399511575Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399533233Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34986)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.39955629Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:37.399586937Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:37.399616804Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:37.399689526Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51926)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.399719843Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.39974385Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.400183364Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"2","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.400211111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"0","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.400762903Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"0","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.400902609Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"2","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.401328734Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:37.401414145Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.400422901Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44112)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.400509312Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.401693748Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.402449358Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[0]R flush_numbers[0..12]: [320, 315, 320, 313, 310, 316, 309, 316, 284, 313, 315, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403025218Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[0]R generation[0..12]: [186, 184, 186, 183, 182, 185, 181, 185, 171, 183, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403046445Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403062014Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[1]R flush_numbers[0..12]: [320, 315, 320, 313, 310, 316, 309, 316, 284, 313, 315, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403075973Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[1]R generation[0..12]: [186, 184, 186, 183, 182, 185, 181, 185, 171, 183, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.4030962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403119208Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[2]R flush_numbers[0..12]: [320, 315, 320, 313, 310, 316, 309, 316, 284, 313, 315, 313]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403143525Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[2]R generation[0..12]: [186, 184, 186, 183, 182, 185, 181, 185, 171, 183, 184, 183]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403168313Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.40319739Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403219927Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403244285Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403268702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403284131Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"":"downstairs","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403298269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403318407Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"0","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403349264Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"1","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403373931Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"client":"2","":"downstairs","session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.40339058Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403403528Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"c2a12323-f7df-43fc-b182-296f260c3008 is now active with session: a82ff958-ea8c-429e-95bd-c5e4a4ad712f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403423456Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403445884Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944,"session_id":"a82ff958-ea8c-429e-95bd-c5e4a4ad712f"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.403465321Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944} {"msg":"Activated sub_volume c2a12323-f7df-43fc-b182-296f260c3008","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:47.40347691Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 297 len 4096 data: 6 02/30 Read block 338 len 16384 03/30 Read block 304 len 36864 04/30 Read block 412 len 36864 05/30 Read block 148 len 40960 06/30 Flush 07/30 Write block 47 len 28672 data: 18 18 19 20 18 20 19 08/30 Write block 581 len 16384 data: 17 16 14 16 09/30 Read block 503 len 4096 10/30 Write block 22 len 32768 data: 12 13 12 12 11 10 9 14 11/30 Write block 577 len 8192 data: 13 15 12/30 Write block 258 len 8192 data: 11 9 13/30 Read block 464 len 40960 14/30 Write block 318 len 28672 data: 12 14 12 12 11 12 15 15/30 Write block 289 len 16384 data: 13 10 10 13 16/30 Read block 154 len 20480 17/30 Flush 18/30 Write block 119 len 32768 data: 13 14 17 16 17 14 12 11 19/30 Read block 590 len 8192 20/30 Read block 51 len 36864 21/30 Write block 219 len 20480 data: 23 22 20 17 18 22/30 Read block 26 len 4096 23/30 Flush 24/30 Write block 572 len 12288 data: 12 13 12 25/30 Read block 301 len 16384 26/30 Read block 273 len 36864 27/30 Write block 401 len 12288 data: 18 20 20 28/30 Read block 547 len 28672 29/30 Write block 441 len 12288 data: 11 10 12 30/30 Write block 40 len 24576 data: 11 14 15 17 17 16 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 3 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:47.423285499Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1944} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 320 320 320 F F F 1 020-039 188 188 188 323 323 323 F F F 2 040-059 188 188 188 323 323 323 T T T 3 060-079 183 183 183 313 313 313 F F F 4 080-099 182 182 182 310 310 310 F F F 5 100-119 188 188 188 324 324 324 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 183 183 183 313 313 313 F F F 10 200-219 188 188 188 324 324 324 F F F 11 220-239 188 188 188 324 324 324 F F F 12 240-259 188 188 188 323 323 323 F F F 13 260-279 186 186 186 318 318 318 F F F 14 280-299 188 188 188 323 323 323 F F F 15 300-319 188 188 188 323 323 323 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 182 182 182 311 311 311 F F F 18 360-379 183 183 183 313 313 313 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 186 186 186 320 320 320 T T T 21 420-439 184 184 184 315 315 315 F F F 22 440-459 186 186 186 320 320 320 T T T 23 460-479 185 185 185 316 316 316 F F F 24 480-499 186 186 186 320 320 320 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 184 184 184 315 315 315 F F F 28 560-579 188 188 188 323 323 323 T T T 29 580-599 188 188 188 323 323 323 F F F Max gen: 188, Max flush: 324 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 320 320 320 F F F 1 020-039 188 188 188 323 323 323 F F F 2 040-059 189 189 189 325 325 325 F F F 3 060-079 183 183 183 313 313 313 F F F 4 080-099 182 182 182 310 310 310 F F F 5 100-119 188 188 188 324 324 324 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 183 183 183 313 313 313 F F F 10 200-219 188 188 188 324 324 324 F F F 11 220-239 188 188 188 324 324 324 F F F 12 240-259 188 188 188 323 323 323 F F F 13 260-279 186 186 186 318 318 318 F F F 14 280-299 188 188 188 323 323 323 F F F 15 300-319 188 188 188 323 323 323 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 182 182 182 311 311 311 F F F 18 360-379 183 183 183 313 313 313 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 189 189 189 325 325 325 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 186 186 186 320 320 320 F F F 25 500-519 182 182 182 311 311 311 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 184 184 184 315 315 315 F F F 28 560-579 189 189 189 325 325 325 F F F 29 580-599 188 188 188 323 323 323 F F F Max gen: 189, Max flush: 325 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.662578871Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.706679548Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.707311841Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.707345978Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.707359906Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.707374125Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952} {"msg":"Crucible de201b09-90ae-49c8-af56-3047b4460c8c has session id: ebd631cf-0030-4453-9909-12ea1a4e4e71","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.707941435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Upstairs opts: Upstairs UUID: de201b09-90ae-49c8-af56-3047b4460c8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.707978951Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Crucible stats registered with UUID: de201b09-90ae-49c8-af56-3047b4460c8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708011518Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:51.708029546Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708449132Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708471279Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952} {"msg":"de201b09-90ae-49c8-af56-3047b4460c8c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708540662Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.70855854Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"0","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708587437Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"0","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708610985Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"1","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708637942Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"1","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708666409Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"2","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708695266Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"2","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708719233Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.70874705Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708778787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"ds_connection connected from Ok(127.0.0.1:37816)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708806084Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"ds_connection connected from Ok(127.0.0.1:47511)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708921722Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"ds_connection connected from Ok(127.0.0.1:59978)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.708958308Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.709476864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"2","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.709505471Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"0","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.710130215Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"2","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.710242284Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"0","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.710632743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.710831642Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.711586902Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"1","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712079371Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"1","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[0]R flush_numbers[0..12]: [320, 323, 325, 313, 310, 324, 324, 316, 284, 313, 324, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712570109Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[0]R generation[0..12]: [186, 188, 189, 183, 182, 188, 188, 185, 171, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712589487Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712603755Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[1]R flush_numbers[0..12]: [320, 323, 325, 313, 310, 324, 324, 316, 284, 313, 324, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712620374Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[1]R generation[0..12]: [186, 188, 189, 183, 182, 188, 188, 185, 171, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712634262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712647881Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[2]R flush_numbers[0..12]: [320, 323, 325, 313, 310, 324, 324, 316, 284, 313, 324, 324]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712661209Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[2]R generation[0..12]: [186, 188, 189, 183, 182, 188, 188, 185, 171, 183, 188, 188]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712674288Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712687787Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712700155Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712712794Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712725343Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712737541Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"":"downstairs","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.71275039Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712761979Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"0","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712774667Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"1","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712787346Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"client":"2","":"downstairs","session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712802564Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-03-18T15:29:51.712814753Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"de201b09-90ae-49c8-af56-3047b4460c8c is now active with session: ebd631cf-0030-4453-9909-12ea1a4e4e71","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712835111Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.71284678Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952,"session_id":"ebd631cf-0030-4453-9909-12ea1a4e4e71"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712858459Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952} {"msg":"Activated sub_volume de201b09-90ae-49c8-af56-3047b4460c8c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:51.712869188Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 87 len 16384 data: 11 10 10 8 02/30 Write block 580 len 12288 data: 13 18 17 03/30 Write block 278 len 28672 data: 14 18 18 18 19 20 22 04/30 Read block 391 len 24576 05/30 Flush 06/30 Write block 543 len 4096 data: 20 07/30 Flush 08/30 Write block 87 len 20480 data: 12 11 11 9 9 09/30 Write block 357 len 28672 data: 11 9 10 10 9 9 10 10/30 Flush 11/30 Write block 57 len 40960 data: 15 15 13 12 10 12 13 14 15 16 12/30 Read block 148 len 32768 13/30 Write block 18 len 28672 data: 12 12 11 12 13 14 13 14/30 Write block 219 len 32768 data: 24 23 21 18 19 16 15 12 15/30 Read block 459 len 16384 16/30 Write block 76 len 20480 data: 9 15 13 13 15 17/30 Write block 83 len 12288 data: 11 13 12 18/30 Read block 424 len 20480 19/30 Write block 2 len 8192 data: 8 10 20/30 Write block 90 len 20480 data: 10 10 10 10 8 21/30 Read block 131 len 36864 22/30 Write block 518 len 8192 data: 21 20 23/30 Read block 11 len 4096 24/30 Write block 197 len 32768 data: 17 14 14 16 14 12 12 10 25/30 Flush 26/30 Write block 490 len 4096 data: 13 27/30 Read block 344 len 32768 28/30 Read block 127 len 40960 29/30 Read block 363 len 32768 30/30 Read block 309 len 36864 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:51.7326433Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1952} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 329 329 329 F F F 1 020-039 190 190 190 329 329 329 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 190 190 190 329 329 329 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 188 188 188 324 324 324 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 190 190 190 329 329 329 F F F 10 200-219 190 190 190 329 329 329 F F F 11 220-239 190 190 190 329 329 329 F F F 12 240-259 188 188 188 323 323 323 F F F 13 260-279 190 190 190 326 326 326 F F F 14 280-299 190 190 190 326 326 326 F F F 15 300-319 188 188 188 323 323 323 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 190 190 190 328 328 328 F F F 18 360-379 190 190 190 328 328 328 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 189 189 189 325 325 325 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 186 186 186 320 320 320 T T T 25 500-519 190 190 190 329 329 329 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 190 190 190 327 327 327 F F F 28 560-579 189 189 189 325 325 325 F F F 29 580-599 190 190 190 326 326 326 F F F Max gen: 190, Max flush: 329 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 329 329 329 F F F 1 020-039 190 190 190 329 329 329 F F F 2 040-059 190 190 190 329 329 329 F F F 3 060-079 190 190 190 329 329 329 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 188 188 188 324 324 324 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 171 171 171 284 284 284 F F F 9 180-199 190 190 190 329 329 329 F F F 10 200-219 190 190 190 329 329 329 F F F 11 220-239 190 190 190 329 329 329 F F F 12 240-259 188 188 188 323 323 323 F F F 13 260-279 190 190 190 326 326 326 F F F 14 280-299 190 190 190 326 326 326 F F F 15 300-319 188 188 188 323 323 323 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 190 190 190 328 328 328 F F F 18 360-379 190 190 190 328 328 328 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 189 189 189 325 325 325 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 191 191 191 330 330 330 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 190 190 190 327 327 327 F F F 28 560-579 189 189 189 325 325 325 F F F 29 580-599 190 190 190 326 326 326 F F F Max gen: 191, Max flush: 330 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:55.946085739Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:55.988194855Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:55.988562406Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.019412165Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.0199413Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.019982715Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.020000553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.020018682Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960} {"msg":"Crucible 944dd5d2-277d-452a-b3d6-403cbd47bcfc has session id: 1f0f54ec-5981-43c2-ad81-673f0c332688","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.020551046Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Upstairs opts: Upstairs UUID: 944dd5d2-277d-452a-b3d6-403cbd47bcfc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.020586102Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Crucible stats registered with UUID: 944dd5d2-277d-452a-b3d6-403cbd47bcfc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.020601361Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:56.020613809Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.02108558Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021108727Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960} {"msg":"944dd5d2-277d-452a-b3d6-403cbd47bcfc active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021172461Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021189519Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"0","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021220345Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"0","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021234394Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"1","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021248282Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"1","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021261251Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"2","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.02127491Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"2","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021287798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021326444Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021341553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"ds_connection connected from Ok(127.0.0.1:43768)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021381959Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"ds_connection connected from Ok(127.0.0.1:34267)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.021416675Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"ds_connection connected from Ok(127.0.0.1:58831)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.02156043Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.022094214Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"1","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.022120561Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"0","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.022135869Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"2","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.022745625Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"2","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.022776602Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"0","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.02279247Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"1","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023377639Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023403726Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[0]R flush_numbers[0..12]: [329, 329, 329, 329, 329, 324, 324, 316, 284, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023433253Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 190, 190, 188, 188, 185, 171, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023448801Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.02346293Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[1]R flush_numbers[0..12]: [329, 329, 329, 329, 329, 324, 324, 316, 284, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023476718Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 190, 190, 188, 188, 185, 171, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023490287Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023503996Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[2]R flush_numbers[0..12]: [329, 329, 329, 329, 329, 324, 324, 316, 284, 329, 329, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023517234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 190, 190, 188, 188, 185, 171, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023530553Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023544011Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023560919Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023574238Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023586887Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023599286Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"":"downstairs","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023612054Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023623693Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"0","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023643261Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"1","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023656189Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"client":"2","":"downstairs","session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023669058Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023681037Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"944dd5d2-277d-452a-b3d6-403cbd47bcfc is now active with session: 1f0f54ec-5981-43c2-ad81-673f0c332688","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023693476Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023705184Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960,"session_id":"1f0f54ec-5981-43c2-ad81-673f0c332688"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023717143Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960} {"msg":"Activated sub_volume 944dd5d2-277d-452a-b3d6-403cbd47bcfc","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:29:56.023728062Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 21 len 12288 data: 13 14 15 02/30 Read block 110 len 40960 03/30 Flush 04/30 Write block 290 len 16384 data: 11 11 14 13 05/30 Write block 169 len 32768 data: 9 11 10 11 11 15 14 10 06/30 Write block 576 len 24576 data: 12 14 16 14 14 19 07/30 Read block 349 len 12288 08/30 Write block 296 len 8192 data: 8 7 09/30 Flush 10/30 Write block 189 len 40960 data: 9 9 14 14 15 14 16 19 18 15 11/30 Write block 344 len 8192 data: 10 12 12/30 Write block 185 len 24576 data: 13 12 10 8 10 10 13/30 Read block 75 len 8192 14/30 Write block 9 len 36864 data: 16 16 16 16 14 15 15 14 15 15/30 Write block 578 len 20480 data: 17 15 15 20 18 16/30 Read block 261 len 40960 17/30 Write block 209 len 24576 data: 14 15 16 16 18 20 18/30 Read block 493 len 36864 19/30 Read block 585 len 24576 20/30 Flush 21/30 Write block 45 len 24576 data: 17 16 19 19 20 21 22/30 Read block 527 len 24576 23/30 Read block 64 len 36864 24/30 Write block 310 len 4096 data: 7 25/30 Read block 222 len 28672 26/30 Read block 538 len 28672 27/30 Read block 591 len 36864 28/30 Write block 346 len 36864 data: 11 12 11 11 10 10 9 9 12 29/30 Write block 105 len 40960 data: 15 15 14 16 13 10 10 11 9 10 30/30 Write block 375 len 20480 data: 11 13 12 15 15 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 6 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:29:56.043294426Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1960} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 333 333 333 F F F 1 020-039 192 192 192 331 331 331 F F F 2 040-059 190 190 190 329 329 329 T T T 3 060-079 190 190 190 329 329 329 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 188 188 188 324 324 324 T T T 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 192 192 192 332 332 332 F F F 9 180-199 192 192 192 333 333 333 F F F 10 200-219 192 192 192 333 333 333 F F F 11 220-239 190 190 190 329 329 329 F F F 12 240-259 188 188 188 323 323 323 F F F 13 260-279 190 190 190 326 326 326 F F F 14 280-299 192 192 192 332 332 332 F F F 15 300-319 188 188 188 323 323 323 T T T 16 320-339 188 188 188 323 323 323 F F F 17 340-359 192 192 192 333 333 333 T T T 18 360-379 190 190 190 328 328 328 T T T 19 380-399 183 183 183 313 313 313 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 189 189 189 325 325 325 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 191 191 191 330 330 330 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 190 190 190 327 327 327 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 192 192 192 333 333 333 F F F Max gen: 192, Max flush: 333 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 333 333 333 F F F 1 020-039 192 192 192 331 331 331 F F F 2 040-059 193 193 193 334 334 334 F F F 3 060-079 190 190 190 329 329 329 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 193 193 193 334 334 334 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 192 192 192 332 332 332 F F F 9 180-199 192 192 192 333 333 333 F F F 10 200-219 192 192 192 333 333 333 F F F 11 220-239 190 190 190 329 329 329 F F F 12 240-259 188 188 188 323 323 323 F F F 13 260-279 190 190 190 326 326 326 F F F 14 280-299 192 192 192 332 332 332 F F F 15 300-319 193 193 193 334 334 334 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 193 193 193 334 334 334 F F F 18 360-379 193 193 193 334 334 334 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 189 189 189 325 325 325 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 191 191 191 330 330 330 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 190 190 190 327 327 327 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 192 192 192 333 333 333 F F F Max gen: 193, Max flush: 334 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.312422156Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.357338207Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.358081719Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.358116206Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.358130624Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.358145023Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968} {"msg":"Crucible 397acdca-13f2-43cc-974e-be27c20f686c has session id: 17907ded-e34b-4593-813b-90b1e5d31c48","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.358656729Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Upstairs opts: Upstairs UUID: 397acdca-13f2-43cc-974e-be27c20f686c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.358782446Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Crucible stats registered with UUID: 397acdca-13f2-43cc-974e-be27c20f686c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.358811603Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:00.358825961Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359341237Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359364535Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968} {"msg":"397acdca-13f2-43cc-974e-be27c20f686c active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359440036Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359459825Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"0","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359488242Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"0","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.35950284Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"1","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359518888Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"1","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359542386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"2","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359606769Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"2","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359647485Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359687331Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359713588Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"ds_connection connected from Ok(127.0.0.1:34335)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359742495Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"ds_connection connected from Ok(127.0.0.1:47734)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359769952Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"ds_connection connected from Ok(127.0.0.1:40752)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.359861742Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.360489436Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"2","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.360528262Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"0","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.361176744Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"0","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.36121895Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"2","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.361250277Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"1","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.361752814Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.361860702Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"1","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.361896638Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"[0]R flush_numbers[0..12]: [333, 331, 334, 329, 329, 334, 324, 316, 332, 333, 333, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362470668Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"[0]R generation[0..12]: [192, 192, 193, 190, 190, 193, 188, 185, 192, 192, 192, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362506574Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362526862Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"[1]R flush_numbers[0..12]: [333, 331, 334, 329, 329, 334, 324, 316, 332, 333, 333, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.36254885Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"[1]R generation[0..12]: [192, 192, 193, 190, 190, 193, 188, 185, 192, 192, 192, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362568168Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362585336Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"[2]R flush_numbers[0..12]: [333, 331, 334, 329, 329, 334, 324, 316, 332, 333, 333, 329]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362601864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"[2]R generation[0..12]: [192, 192, 193, 190, 190, 193, 188, 185, 192, 192, 192, 190]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362618423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.3626439Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362659758Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362675657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362691235Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362706813Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"":"downstairs","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362722632Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.36273707Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"0","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362753108Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"1","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362769097Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"client":"2","":"downstairs","session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362785295Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362802423Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"397acdca-13f2-43cc-974e-be27c20f686c is now active with session: 17907ded-e34b-4593-813b-90b1e5d31c48","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-03-18T15:30:00.362818282Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.36283555Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968,"session_id":"17907ded-e34b-4593-813b-90b1e5d31c48"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362850468Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968} {"msg":"Activated sub_volume 397acdca-13f2-43cc-974e-be27c20f686c","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:00.362864067Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 305 len 36864 data: 11 11 11 8 8 8 10 11 12 02/30 Read block 58 len 32768 03/30 Read block 383 len 32768 04/30 Write block 480 len 12288 data: 12 16 15 05/30 Read block 127 len 16384 06/30 Flush 07/30 Write block 239 len 28672 data: 13 11 12 13 14 12 12 08/30 Read block 162 len 8192 09/30 Write block 72 len 20480 data: 10 11 12 11 10 10/30 Read block 376 len 20480 11/30 Read block 225 len 16384 12/30 Read block 138 len 32768 13/30 Read block 62 len 28672 14/30 Read block 248 len 36864 15/30 Read block 106 len 28672 16/30 Read block 214 len 16384 17/30 Read block 503 len 40960 18/30 Write block 108 len 12288 data: 17 14 11 19/30 Read block 550 len 36864 20/30 Read block 204 len 32768 21/30 Write block 446 len 16384 data: 15 15 14 15 22/30 Flush 23/30 Write block 228 len 8192 data: 10 10 24/30 Read block 376 len 12288 25/30 Write block 364 len 32768 data: 10 13 13 14 14 15 16 15 26/30 Read block 307 len 40960 27/30 Write block 358 len 20480 data: 10 11 11 10 10 28/30 Read block 447 len 8192 29/30 Read block 376 len 24576 30/30 Read block 194 len 8192 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:00.384599324Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1968} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 333 333 333 F F F 1 020-039 192 192 192 331 331 331 F F F 2 040-059 193 193 193 334 334 334 F F F 3 060-079 194 194 194 336 336 336 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 194 194 194 336 336 336 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 192 192 192 332 332 332 F F F 9 180-199 192 192 192 333 333 333 F F F 10 200-219 192 192 192 333 333 333 F F F 11 220-239 194 194 194 336 336 336 T T T 12 240-259 194 194 194 336 336 336 F F F 13 260-279 190 190 190 326 326 326 F F F 14 280-299 192 192 192 332 332 332 F F F 15 300-319 194 194 194 335 335 335 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 193 193 193 334 334 334 T T T 18 360-379 193 193 193 334 334 334 T T T 19 380-399 183 183 183 313 313 313 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 194 194 194 336 336 336 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 194 194 194 335 335 335 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 190 190 190 327 327 327 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 192 192 192 333 333 333 F F F Max gen: 194, Max flush: 336 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 333 333 333 F F F 1 020-039 192 192 192 331 331 331 F F F 2 040-059 193 193 193 334 334 334 F F F 3 060-079 194 194 194 336 336 336 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 194 194 194 336 336 336 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 192 192 192 332 332 332 F F F 9 180-199 192 192 192 333 333 333 F F F 10 200-219 192 192 192 333 333 333 F F F 11 220-239 195 195 195 337 337 337 F F F 12 240-259 194 194 194 336 336 336 F F F 13 260-279 190 190 190 326 326 326 F F F 14 280-299 192 192 192 332 332 332 F F F 15 300-319 194 194 194 335 335 335 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 195 195 195 337 337 337 F F F 18 360-379 195 195 195 337 337 337 F F F 19 380-399 183 183 183 313 313 313 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 184 184 184 315 315 315 F F F 22 440-459 194 194 194 336 336 336 F F F 23 460-479 185 185 185 316 316 316 F F F 24 480-499 194 194 194 335 335 335 F F F 25 500-519 190 190 190 329 329 329 F F F 26 520-539 181 181 181 309 309 309 F F F 27 540-559 190 190 190 327 327 327 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 192 192 192 333 333 333 F F F Max gen: 195, Max flush: 337 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.618084438Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:14.661197389Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.661556441Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69345006Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.694153797Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.694189453Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.694203011Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69421795Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976} {"msg":"Crucible 77b3c951-fee2-4c0e-8eb4-152938607def has session id: e3e77e70-0958-4c0e-95ea-f9612b447cd9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69478531Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Upstairs opts: Upstairs UUID: 77b3c951-fee2-4c0e-8eb4-152938607def, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.694827426Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Crucible stats registered with UUID: 77b3c951-fee2-4c0e-8eb4-152938607def","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.694852683Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:14.694873411Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695308265Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695334833Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976} {"msg":"77b3c951-fee2-4c0e-8eb4-152938607def active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695403435Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695421363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"0","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69545053Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"0","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695470118Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"1","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695484657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"1","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695498225Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"2","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695518743Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"2","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695541071Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695564848Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695588176Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38296)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695613013Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48164)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.6956401Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45556)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.695667667Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.696464864Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"1","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69650244Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"2","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.696625747Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"0","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.697203416Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"1","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.697238273Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"2","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.697368779Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"0","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69783536Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.697865067Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[0]R flush_numbers[0..12]: [333, 331, 334, 336, 329, 336, 324, 316, 332, 333, 333, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69793058Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[0]R generation[0..12]: [192, 192, 193, 194, 190, 194, 188, 185, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.697951398Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.697967166Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[1]R flush_numbers[0..12]: [333, 331, 334, 336, 329, 336, 324, 316, 332, 333, 333, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.697981615Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[1]R generation[0..12]: [192, 192, 193, 194, 190, 194, 188, 185, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.697995983Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698010661Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[2]R flush_numbers[0..12]: [333, 331, 334, 336, 329, 336, 324, 316, 332, 333, 333, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69802575Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[2]R generation[0..12]: [192, 192, 193, 194, 190, 194, 188, 185, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698039608Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698053807Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698067465Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698085153Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Next flush: 338","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698098522Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698111841Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"":"downstairs","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698124619Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698136328Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"0","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698149347Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"1","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Transition from WaitQuorum to Active","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698170664Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"client":"2","":"downstairs","session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698184833Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698197582Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"77b3c951-fee2-4c0e-8eb4-152938607def is now active with session: e3e77e70-0958-4c0e-95ea-f9612b447cd9","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.69821036Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698222019Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976,"session_id":"e3e77e70-0958-4c0e-95ea-f9612b447cd9"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698233798Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976} {"msg":"Activated sub_volume 77b3c951-fee2-4c0e-8eb4-152938607def","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:14.698244657Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 28672 data: 17 20 20 21 22 19 21 02/30 Read block 226 len 4096 03/30 Flush 04/30 Write block 505 len 4096 data: 15 05/30 Read block 579 len 4096 06/30 Read block 248 len 8192 07/30 Read block 144 len 36864 08/30 Flush 09/30 Write block 462 len 28672 data: 18 19 18 16 20 17 18 10/30 Write block 295 len 24576 data: 10 9 8 11 11 15 11/30 Read block 440 len 12288 12/30 Write block 504 len 16384 data: 14 16 14 14 13/30 Read block 217 len 36864 14/30 Flush 15/30 Write block 33 len 12288 data: 18 17 18 16/30 Read block 153 len 40960 17/30 Write block 554 len 8192 data: 15 18 18/30 Write block 518 len 36864 data: 22 21 19 18 16 13 10 11 14 19/30 Read block 314 len 12288 20/30 Read block 146 len 28672 21/30 Read block 368 len 28672 22/30 Write block 538 len 8192 data: 16 17 23/30 Write block 433 len 28672 data: 18 19 18 16 14 12 9 24/30 Read block 371 len 40960 25/30 Read block 558 len 24576 26/30 Flush 27/30 Write block 378 len 12288 data: 16 16 14 28/30 Write block 357 len 40960 data: 12 11 12 12 11 11 11 11 14 14 29/30 Write block 34 len 20480 data: 18 19 17 18 14 30/30 Write block 246 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 1 1 1 3 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:14.717731809Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1976} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 333 333 333 F F F 1 020-039 196 196 196 341 341 341 T T T 2 040-059 196 196 196 338 338 338 F F F 3 060-079 194 194 194 336 336 336 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 194 194 194 336 336 336 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 192 192 192 332 332 332 F F F 9 180-199 192 192 192 333 333 333 F F F 10 200-219 192 192 192 333 333 333 F F F 11 220-239 195 195 195 337 337 337 F F F 12 240-259 194 194 194 336 336 336 T T T 13 260-279 190 190 190 326 326 326 F F F 14 280-299 196 196 196 340 340 340 F F F 15 300-319 196 196 196 340 340 340 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 195 195 195 337 337 337 T T T 18 360-379 195 195 195 337 337 337 T T T 19 380-399 183 183 183 313 313 313 T T T 20 400-419 189 189 189 325 325 325 F F F 21 420-439 196 196 196 341 341 341 F F F 22 440-459 194 194 194 336 336 336 F F F 23 460-479 196 196 196 340 340 340 F F F 24 480-499 194 194 194 335 335 335 F F F 25 500-519 196 196 196 341 341 341 F F F 26 520-539 196 196 196 341 341 341 F F F 27 540-559 196 196 196 341 341 341 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 192 192 192 333 333 333 F F F Max gen: 196, Max flush: 341 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 333 333 333 F F F 1 020-039 197 197 197 342 342 342 F F F 2 040-059 196 196 196 338 338 338 F F F 3 060-079 194 194 194 336 336 336 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 194 194 194 336 336 336 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 192 192 192 332 332 332 F F F 9 180-199 192 192 192 333 333 333 F F F 10 200-219 192 192 192 333 333 333 F F F 11 220-239 195 195 195 337 337 337 F F F 12 240-259 197 197 197 342 342 342 F F F 13 260-279 190 190 190 326 326 326 F F F 14 280-299 196 196 196 340 340 340 F F F 15 300-319 196 196 196 340 340 340 F F F 16 320-339 188 188 188 323 323 323 F F F 17 340-359 197 197 197 342 342 342 F F F 18 360-379 197 197 197 342 342 342 F F F 19 380-399 197 197 197 342 342 342 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 196 196 196 341 341 341 F F F 22 440-459 194 194 194 336 336 336 F F F 23 460-479 196 196 196 340 340 340 F F F 24 480-499 194 194 194 335 335 335 F F F 25 500-519 196 196 196 341 341 341 F F F 26 520-539 196 196 196 341 341 341 F F F 27 540-559 196 196 196 341 341 341 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 192 192 192 333 333 333 F F F Max gen: 197, Max flush: 342 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:18.984340312Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.028007274Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.028590183Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a70f3a885a6160c78d68c43ecf4da40ad9a7b327\",\n git_commit_timestamp: \"2025-03-18T14:56:23.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.0286231Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.028637258Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.028651866Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984} {"msg":"Crucible 13ed0b24-cdce-4f1f-88f5-125cfcec9f3f has session id: 8e792eee-548e-4e13-8f5a-3e6850224907","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029257063Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Upstairs opts: Upstairs UUID: 13ed0b24-cdce-4f1f-88f5-125cfcec9f3f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029295799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Crucible stats registered with UUID: 13ed0b24-cdce-4f1f-88f5-125cfcec9f3f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029319186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:19.029338794Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029793577Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029815724Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984} {"msg":"13ed0b24-cdce-4f1f-88f5-125cfcec9f3f active request set","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029876628Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029893016Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"0","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029919883Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"0","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029933962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"1","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.02994778Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"1","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.029969078Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"client set_active_request while in Connecting { state: Start { auto_promote: true }, mode: New }; waiting...","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.030016393Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.03004825Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.030075947Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.030103304Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:19.030131831Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:19.030159458Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"ds_connection connected from Ok(127.0.0.1:58397)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.030189145Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:19.030217962Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"ds_connection connected from Ok(127.0.0.1:39126)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.030241219Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.030267007Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.030798521Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"1","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.030830937Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"0","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.031421286Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"0","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.031557511Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"1","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.031987236Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:19.032096115Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.031113772Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"ds_connection connected from Ok(127.0.0.1:60402)","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.031198843Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"version negotiation from state Connecting { state: WaitForPromote, mode: New }","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.0322679Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.032806164Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[0]R flush_numbers[0..12]: [333, 342, 338, 336, 329, 336, 324, 316, 332, 333, 333, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.03341118Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[0]R generation[0..12]: [192, 197, 196, 194, 190, 194, 188, 185, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033437128Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033453806Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[1]R flush_numbers[0..12]: [333, 342, 338, 336, 329, 336, 324, 316, 332, 333, 333, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033467484Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[1]R generation[0..12]: [192, 197, 196, 194, 190, 194, 188, 185, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033480923Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033494842Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[2]R flush_numbers[0..12]: [333, 342, 338, 336, 329, 336, 324, 316, 332, 333, 333, 337]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.03350837Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[2]R generation[0..12]: [192, 197, 196, 194, 190, 194, 188, 185, 192, 192, 192, 195]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033521799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033535357Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033547796Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033560655Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Next flush: 343","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033573363Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033585782Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"":"downstairs","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033598641Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033612269Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"0","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033644186Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"1","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033658234Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"client":"2","":"downstairs","session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033671253Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033683272Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"13ed0b24-cdce-4f1f-88f5-125cfcec9f3f is now active with session: 8e792eee-548e-4e13-8f5a-3e6850224907","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.03369705Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033708799Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984,"session_id":"8e792eee-548e-4e13-8f5a-3e6850224907"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033722228Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984} {"msg":"Activated sub_volume 13ed0b24-cdce-4f1f-88f5-125cfcec9f3f","v":0,"name":"crucible","level":30,"time":"2025-03-18T15:30:29.033741386Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 188 len 36864 data: 9 11 11 15 15 16 15 17 20 02/30 Read block 19 len 24576 03/30 Write block 317 len 28672 data: 14 13 15 13 13 12 13 04/30 Read block 364 len 8192 05/30 Write block 522 len 28672 data: 17 14 11 12 15 13 15 06/30 Read block 459 len 8192 07/30 Read block 414 len 8192 08/30 Read block 174 len 12288 09/30 Read block 259 len 24576 10/30 Read block 303 len 24576 11/30 Read block 41 len 8192 12/30 Read block 359 len 4096 13/30 Read block 56 len 24576 14/30 Read block 285 len 36864 15/30 Write block 547 len 8192 data: 14 14 16/30 Read block 107 len 20480 17/30 Write block 390 len 28672 data: 14 16 19 19 18 19 21 18/30 Read block 124 len 36864 19/30 Flush 20/30 Write block 114 len 20480 data: 11 10 9 9 11 21/30 Write block 245 len 36864 data: 13 13 15 15 14 12 12 15 13 22/30 Read block 53 len 36864 23/30 Write block 249 len 36864 data: 15 13 13 16 14 13 14 10 10 24/30 Read block 304 len 24576 25/30 Read block 62 len 12288 26/30 Read block 426 len 36864 27/30 Read block 403 len 36864 28/30 Write block 104 len 24576 data: 19 16 16 15 18 15 29/30 Read block 242 len 28672 30/30 Read block 27 len 28672 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-03-18T15:30:29.056512983Z","hostname":"w-01JPMWYJD1AD31EBCX6CCA123C","pid":1984} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 333 333 333 F F F 1 020-039 197 197 197 342 342 342 F F F 2 040-059 196 196 196 338 338 338 F F F 3 060-079 194 194 194 336 336 336 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 194 194 194 336 336 336 T T T 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 192 192 192 332 332 332 F F F 9 180-199 198 198 198 343 343 343 F F F 10 200-219 192 192 192 333 333 333 F F F 11 220-239 195 195 195 337 337 337 F F F 12 240-259 197 197 197 342 342 342 T T T 13 260-279 190 190 190 326 326 326 F F F 14 280-299 196 196 196 340 340 340 F F F 15 300-319 198 198 198 343 343 343 F F F 16 320-339 198 198 198 343 343 343 F F F 17 340-359 197 197 197 342 342 342 F F F 18 360-379 197 197 197 342 342 342 F F F 19 380-399 198 198 198 343 343 343 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 196 196 196 341 341 341 F F F 22 440-459 194 194 194 336 336 336 F F F 23 460-479 196 196 196 340 340 340 F F F 24 480-499 194 194 194 335 335 335 F F F 25 500-519 196 196 196 341 341 341 F F F 26 520-539 198 198 198 343 343 343 F F F 27 540-559 198 198 198 343 343 343 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 192 192 192 333 333 333 F F F Max gen: 198, Max flush: 343 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 333 333 333 F F F 1 020-039 197 197 197 342 342 342 F F F 2 040-059 196 196 196 338 338 338 F F F 3 060-079 194 194 194 336 336 336 F F F 4 080-099 190 190 190 329 329 329 F F F 5 100-119 199 199 199 344 344 344 F F F 6 120-139 188 188 188 324 324 324 F F F 7 140-159 185 185 185 316 316 316 F F F 8 160-179 192 192 192 332 332 332 F F F 9 180-199 198 198 198 343 343 343 F F F 10 200-219 192 192 192 333 333 333 F F F 11 220-239 195 195 195 337 337 337 F F F 12 240-259 199 199 199 344 344 344 F F F 13 260-279 190 190 190 326 326 326 F F F 14 280-299 196 196 196 340 340 340 F F F 15 300-319 198 198 198 343 343 343 F F F 16 320-339 198 198 198 343 343 343 F F F 17 340-359 197 197 197 342 342 342 F F F 18 360-379 197 197 197 342 342 342 F F F 19 380-399 198 198 198 343 343 343 F F F 20 400-419 189 189 189 325 325 325 F F F 21 420-439 196 196 196 341 341 341 F F F 22 440-459 194 194 194 336 336 336 F F F 23 460-479 196 196 196 340 340 340 F F F 24 480-499 194 194 194 335 335 335 F F F 25 500-519 196 196 196 341 341 341 F F F 26 520-539 198 198 198 343 343 343 F F F 27 540-559 198 198 198 343 343 343 F F F 28 560-579 192 192 192 333 333 333 F F F 29 580-599 192 192 192 333 333 333 F F F Max gen: 199, Max flush: 344 17:54 Test duration Test completed real 17:53.898191629 user 49.881087490 sys 19.110979824 trap 0.062993518 tflt 0.087696935 dflt 0.133007697 kflt 0.006057146 lock 28:27:28.364006370 slp 1:28:07.094170040 lat 5.587334990 stop 0.065566067